Updated version string for the start of development for 2.1.0.

[fs/lustre-release.git] / lustre / utils / lconf
diff --git a/lustre/utils/lconf b/lustre/utils/lconf

index f704c77..bfaa97a 100755 (executable)
--- a/lustre/utils/lconf
+++ b/lustre/utils/lconf
@@ -1,31 +1,51 @@
  #!/usr/bin/env python
  #
  #!/usr/bin/env python
  #
-#  Copyright (C) 2002-2003 Cluster File Systems, Inc.
-#   Authors: Robert Read <rread@clusterfs.com>
-#            Mike Shaver <shaver@clusterfs.com>
-#   This file is part of Lustre, http://www.lustre.org.
+# GPL HEADER START
  #
  #
-#   Lustre is free software; you can redistribute it and/or
-#   modify it under the terms of version 2 of the GNU General Public
-#   License as published by the Free Software Foundation.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  #
  #
-#   Lustre is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 only,
+# as published by the Free Software Foundation.
  #
  #
-#   You should have received a copy of the GNU General Public License
-#   along with Lustre; if not, write to the Free Software
-#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License version 2 for more details (a copy is included
+# in the LICENSE file that accompanied this code).
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; If not, see
+# http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
+# copy of GPLv2].
+#
+# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+# CA 95054 USA or visit www.sun.com if you need additional information or
+# have any questions.
+#
+# GPL HEADER END
+#
+
+#
+# Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# This file is part of Lustre, http://www.lustre.org/
+# Lustre is a trademark of Sun Microsystems, Inc.
+#
+# Author: Robert Read <rread@clusterfs.com>
+# Author: Mike Shaver <shaver@clusterfs.com>
  #
  # lconf - lustre configuration tool
  #
  # lconf is the main driver script for starting and stopping
  # lustre filesystem services.
  #
  #
  # lconf - lustre configuration tool
  #
  # lconf is the main driver script for starting and stopping
  # lustre filesystem services.
  #
-# Based in part on the XML obdctl modifications done by Brian Behlendorf 
+# Based in part on the XML obdctl modifications done by Brian Behlendorf
  
  
-import sys, getopt, types
+import sys, getopt, types, errno
  import string, os, stat, popen2, socket, time, random, fcntl, select
  import re, exceptions, signal, traceback
  import xml.dom.minidom
  import string, os, stat, popen2, socket, time, random, fcntl, select
  import re, exceptions, signal, traceback
  import xml.dom.minidom
@@ -35,7 +55,16 @@ if sys.version[0] == '1':
  else:
      from fcntl import F_GETFL, F_SETFL
  
  else:
      from fcntl import F_GETFL, F_SETFL
  
-PYMOD_DIR = "/usr/lib/lustre/python"
+PYMOD_DIR = ["/usr/lib64/lustre/python", "/usr/lib/lustre/python"]
+PLATFORM = ''
+KEXTPATH = ''
+if string.find(sys.platform, 'linux') != -1:
+    PLATFORM='LINUX'
+elif string.find(sys.platform, 'darwin') != -1:
+    PLATFORM='DARWIN'
+    KEXTPATH='/System/Library/Extensions/'
+else:
+    PLATFORM='Unsupported'
  
  def development_mode():
      base = os.path.dirname(sys.argv[0])
  
  def development_mode():
      base = os.path.dirname(sys.argv[0])
@@ -46,25 +75,23 @@ def development_mode():
  if development_mode():
      sys.path.append('../utils')
  else:
  if development_mode():
      sys.path.append('../utils')
  else:
-    sys.path.append(PYMOD_DIR)
+    sys.path.extend(PYMOD_DIR)
  
  import Lustre
  
  # Global parameters
  MAXTCPBUF = 16777216
  
  import Lustre
  
  # Global parameters
  MAXTCPBUF = 16777216
-DEFAULT_TCPBUF = 8388608
-DEFAULT_PORT = 988
  #
  # Maximum number of devices to search for.
  # (the /dev/loop* nodes need to be created beforehand)
  MAX_LOOP_DEVICES = 256
  #
  # Maximum number of devices to search for.
  # (the /dev/loop* nodes need to be created beforehand)
  MAX_LOOP_DEVICES = 256
-PORTALS_DIR = '../portals'
+PORTALS_DIR = '../lnet'
  
  # Needed to call lconf --record
  
  # Needed to call lconf --record
-CONFIG_FILE = "" 
+CONFIG_FILE = ""
  
  
-# Please keep these in sync with the values in portals/kp30.h
-ptldebug_names = { 
+# Please keep these in sync with the values in lnet/include/libcfs/libcfs.h
+ptldebug_names = {
      "trace" :     (1 << 0),
      "inode" :     (1 << 1),
      "super" :     (1 << 2),
      "trace" :     (1 << 0),
      "inode" :     (1 << 1),
      "super" :     (1 << 2),
@@ -79,7 +106,8 @@ ptldebug_names = {
      "buffs" :     (1 << 11),
      "other" :     (1 << 12),
      "dentry" :    (1 << 13),
      "buffs" :     (1 << 11),
      "other" :     (1 << 12),
      "dentry" :    (1 << 13),
-    "portals" :   (1 << 14),
+    "portals" :   (1 << 14), # deprecated
+    "lnet" :      (1 << 14),
      "page" :      (1 << 15),
      "dlmtrace" :  (1 << 16),
      "error" :     (1 << 17),
      "page" :      (1 << 15),
      "dlmtrace" :  (1 << 16),
      "error" :     (1 << 17),
@@ -90,7 +118,10 @@ ptldebug_names = {
      "reada" :     (1 << 22),
      "mmap" :      (1 << 23),
      "config" :    (1 << 24),
      "reada" :     (1 << 22),
      "mmap" :      (1 << 23),
      "config" :    (1 << 24),
-}
+    "console" :   (1 << 25),
+    "quota" :     (1 << 26),
+    "sec" :       (1 << 27),
+    }
  
  subsystem_names = {
      "undefined" :    (1 << 0),
  
  subsystem_names = {
      "undefined" :    (1 << 0),
@@ -102,22 +133,29 @@ subsystem_names = {
      "log" :          (1 << 6),
      "llite" :        (1 << 7),
      "rpc" :          (1 << 8),
      "log" :          (1 << 6),
      "llite" :        (1 << 7),
      "rpc" :          (1 << 8),
-    "mgmt" :         (1 << 9),
-    "portals" :      (1 << 10),
-    "nal" :          (1 << 11),
+    "lnet" :         (1 << 10),
+    "portals" :      (1 << 10), # deprecated
+    "lnd" :          (1 << 11),
+    "nal" :          (1 << 11), # deprecated
      "pinger" :       (1 << 12),
      "filter" :       (1 << 13),
      "pinger" :       (1 << 12),
      "filter" :       (1 << 13),
-    "ptlbd" :        (1 << 14),
+    "ptlbd" :        (1 << 14), # deprecated
      "echo" :         (1 << 15),
      "ldlm" :         (1 << 16),
      "lov" :          (1 << 17),
      "echo" :         (1 << 15),
      "ldlm" :         (1 << 16),
      "lov" :          (1 << 17),
-    "ptlrouter" :    (1 << 18),
+    "ptlrouter" :    (1 << 18), # deprecated
      "cobd" :         (1 << 19),
      "sm" :           (1 << 20),
      "asobd" :        (1 << 21),
      "cobd" :         (1 << 19),
      "sm" :           (1 << 20),
      "asobd" :        (1 << 21),
-    "confobd" :      (1 << 22),
+    "confobd" :      (1 << 22), # deprecated
      "lmv" :          (1 << 23),
      "cmobd" :        (1 << 24),
      "lmv" :          (1 << 23),
      "cmobd" :        (1 << 24),
+    "sec" :          (1 << 25),
+    "sec" :          (1 << 26),
+    "gss" :          (1 << 27),
+    "gks" :          (1 << 28),
+    "mgc" :          (1 << 29),
+    "mgs" :          (1 << 30),
      }
  
  
      }
  
  
@@ -127,7 +165,7 @@ def cleanup_error(rc):
      if not first_cleanup_error:
          first_cleanup_error = rc
  
      if not first_cleanup_error:
          first_cleanup_error = rc
  
-# ============================================================ 
+# ============================================================
  # debugging and error funcs
  
  def fixme(msg = "this feature"):
  # debugging and error funcs
  
  def fixme(msg = "this feature"):
@@ -149,20 +187,27 @@ def logall(msgs):
          print string.strip(s)
  
  def debug(*args):
          print string.strip(s)
  
  def debug(*args):
+    # apparently, (non)execution of the following line affects mds device
+    # startup order (e.g. two mds's using loopback devices), so always do it.
+    msg = string.join(map(str,args))
      if config.verbose:
      if config.verbose:
-        msg = string.join(map(str,args))
          print msg
  
  # ack, python's builtin int() does not support '0x123' syntax.
  # eval can do it, although what a hack!
  def my_int(s):
          print msg
  
  # ack, python's builtin int() does not support '0x123' syntax.
  # eval can do it, although what a hack!
  def my_int(s):
+    import types
+    if type(s) is types.IntType:
+        return s
      try:
      try:
-        if s[0:2] == '0x':
+        if (s[0:2] == '0x') or (s[0:1] == '0'):
              return eval(s, {}, {})
          else:
              return int(s)
      except SyntaxError, e:
          raise ValueError("not a number")
              return eval(s, {}, {})
          else:
              return int(s)
      except SyntaxError, e:
          raise ValueError("not a number")
+    except TypeError, e:
+        raise ValueError("not a number")
      except NameError, e:
          raise ValueError("not a number")
  
      except NameError, e:
          raise ValueError("not a number")
  
@@ -191,117 +236,6 @@ class CommandError (exceptions.Exception):
          else:
              print self.cmd_err
  
          else:
              print self.cmd_err
  
-
-# ============================================================
-# handle daemons, like the acceptor
-class DaemonHandler:
-    """ Manage starting and stopping a daemon. Assumes daemon manages
-    it's own pid file. """
-
-    def __init__(self, cmd):
-        self.command = cmd
-        self.path =""
-
-    def start(self):
-        if self.running():
-            log(self.command, "already running.")
-        if not self.path:
-            self.path = find_prog(self.command)
-            if not self.path:
-                panic(self.command, "not found.")
-        ret, out = runcmd(self.path +' '+ self.command_line())
-        if ret:
-            raise CommandError(self.path, out, ret)
-
-    def stop(self):
-        if self.running():
-            pid = self.read_pidfile()
-            try:
-               if pid != 1:
-                   log ("killing process", pid)
-                   os.kill(pid, 15)
-               else:
-                   log("was unable to find pid of " + self.command)
-                #time.sleep(1) # let daemon die
-            except OSError, e:
-                log("unable to kill", self.command, e)
-            if self.running():
-                log("unable to kill", self.command)
-
-    def running(self):
-        pid = self.read_pidfile()
-        if pid:
-            try:
-               if pid != 1:
-                   os.kill(pid, 0)
-               else:
-                   log("was unable to find pid of " + self.command)
-            except OSError:
-                self.clean_pidfile()
-            else:
-                return 1
-        return 0
-
-    def read_pidfile(self):
-        try:
-            fp = open(self.pidfile(), 'r')
-           val = fp.read()
-           if val == '':
-               val = '1'
-            pid = int(val)
-            fp.close()
-            return pid
-        except IOError:
-            return 0
-        
-    def clean_pidfile(self):
-        """ Remove a stale pidfile """
-        log("removing stale pidfile:", self.pidfile())
-        try:
-            os.unlink(self.pidfile())
-        except OSError, e:
-            log(self.pidfile(), e)
-            
-class AcceptorHandler(DaemonHandler):
-    def __init__(self, port, net_type):
-        DaemonHandler.__init__(self, "acceptor")
-        self.port = port
-        self.flags = ''
-
-    def pidfile(self):
-        return "/var/run/%s-%d.pid" % (self.command, self.port)
-
-    def command_line(self):
-        return string.join(map(str,(self.flags, self.port)))
-    
-acceptors = {}
-
-# start the acceptors
-def run_acceptors():
-    if config.lctl_dump or config.record:
-        return
-    for port in acceptors.keys():
-        daemon = acceptors[port]
-        if not daemon.running():
-            daemon.start()
-
-def run_one_acceptor(port):
-    if config.lctl_dump or config.record:
-        return
-    if acceptors.has_key(port):
-        daemon = acceptors[port]
-        if not daemon.running():
-            daemon.start()
-    else:
-         panic("run_one_acceptor: No acceptor defined for port:", port)   
-        
-def stop_acceptor(port):
-    if acceptors.has_key(port):
-        daemon = acceptors[port]
-        if daemon.running():
-            daemon.stop()
-        
-
  # ============================================================
  # handle lctl interface
  class LCTLInterface:
  # ============================================================
  # handle lctl interface
  class LCTLInterface:
@@ -325,7 +259,7 @@ class LCTLInterface:
  
      def use_save_file(self, file):
          self.save_file = file
  
      def use_save_file(self, file):
          self.save_file = file
-        
+
      def record(self, dev_name, logname):
          log("Recording log", logname, "on", dev_name)
          self.record_device = dev_name
      def record(self, dev_name, logname):
          log("Recording log", logname, "on", dev_name)
          self.record_device = dev_name
@@ -357,14 +291,13 @@ class LCTLInterface:
      device $%s
      record %s
      %s""" % (self.record_device, self.record_log, cmds)
      device $%s
      record %s
      %s""" % (self.record_device, self.record_log, cmds)
-            
+
          debug("+", cmd_line, cmds)
          if config.noexec: return (0, [])
  
          child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
          debug("+", cmd_line, cmds)
          if config.noexec: return (0, [])
  
          child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
-        child.tochild.write(cmds + "\n")
+        child.tochild.write(cmds + "\nq\n")
          child.tochild.close()
          child.tochild.close()
-#      print "LCTL:", cmds
  
          # From "Python Cookbook" from O'Reilly
          outfile = child.fromchild
  
          # From "Python Cookbook" from O'Reilly
          outfile = child.fromchild
@@ -409,6 +342,12 @@ class LCTLInterface:
              raise CommandError(self.lctl, out, rc)
          return rc, out
  
              raise CommandError(self.lctl, out, rc)
          return rc, out
  
+    def unconfigure_network(self):
+        """get lnet to unreference itself"""
+        cmds = """
+  network unconfigure"""
+        self.run(cmds)
+  
      def clear_log(self, dev, log):
          """ clear an existing log """
          cmds =  """
      def clear_log(self, dev, log):
          """ clear an existing log """
          cmds =  """
@@ -418,73 +357,25 @@ class LCTLInterface:
    quit """ % (dev, log)
          self.run(cmds)
  
    quit """ % (dev, log)
          self.run(cmds)
  
-    def root_squash(self, name, uid, nid):
-        cmds = """
-  device $%s
-  root_squash %s %s
-  quit""" % (name, uid, nid)
-        self.run(cmds)
-
-    def network(self, net, nid):
-        """ set mynid """
-        cmds =  """
-  network %s
-  mynid %s
-  quit """ % (net, nid)
-        self.run(cmds)
-
-    # add an interface
-    def add_interface(self, net, ip, netmask = ""):
-        """ add an interface """
-        cmds = """
-  network %s
-  add_interface %s %s
-  quit """ % (net, ip, netmask)
-        self.run(cmds)
-
-    # delete an interface
-    def del_interface(self, net, ip):
-        """ delete an interface """
-        cmds = """
-  network %s
-  del_interface %s
-  quit """ % (net, ip)
-        self.run(cmds)
-
      # create a new connection
      def add_uuid(self, net_type, uuid, nid):
      # create a new connection
      def add_uuid(self, net_type, uuid, nid):
-        cmds = "\n  add_uuid %s %s %s" %(uuid, nid, net_type)
+        if net_type != 'lnet' and string.find(nid,'@') < 0:
+            nidstr = nid + "@" + net_type
+        else:
+            nidstr = nid
+        cmds = "\n  add_uuid %s %s" %(uuid, nidstr)
          self.run(cmds)
  
          self.run(cmds)
  
-    def add_peer(self, net_type, nid, hostaddr, port):
-        if net_type  in ('tcp','openib','ra') and not config.lctl_dump:
-            cmds =  """
-  network %s
-  add_peer %s %s %d
-  quit""" % (net_type,
-             nid, hostaddr, port )
-            self.run(cmds)
-        elif net_type in ('iib',) and not config.lctl_dump:
-            cmds =  """
-  network %s
-  add_peer %s
-  quit""" % (net_type,
-             nid )
-            self.run(cmds)
-        elif net_type in ('vib',) and not config.lctl_dump:
-            cmds =  """
-  network %s
-  add_peer %s %s
-  quit""" % (net_type,
-             nid, hostaddr )
-            self.run(cmds)
-    
      def connect(self, srv):
      def connect(self, srv):
-        self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
-        if srv.net_type  in ('tcp','openib','iib','vib','ra') and not config.lctl_dump:
-            if srv.hostaddr[0]:
-                hostaddr = string.split(srv.hostaddr[0], '/')[0]
-            self.add_peer(srv.net_type, srv.nid, hostaddr, srv.port)
+        if not srv.nid_uuid:
+            panic('nid_uuid not set for ', srv.net_type, srv.nid)
+        hostaddr = srv.db.get_hostaddr()
+        if len(hostaddr) > 1:
+            panic('multiple --hostaddr for ', srv.nid_uuid, ' not supported')
+        elif len(hostaddr) == 1 and hostaddr[0] != srv.nid:
+            panic('different --hostaddr and --nid  for ', srv.nid_uuid, ' not supported')
+        else:
+            self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
  
      # Recover a device
      def recover(self, dev_name, new_conn):
  
      # Recover a device
      def recover(self, dev_name, new_conn):
@@ -492,78 +383,12 @@ class LCTLInterface:
      device $%s
      recover %s""" %(dev_name, new_conn)
          self.run(cmds)
      device $%s
      recover %s""" %(dev_name, new_conn)
          self.run(cmds)
-                
-    # add a route to a range
-    def add_route(self, net, gw, lo, hi):
-        cmds =  """
-  network %s
-  add_route %s %s %s
-  quit  """ % (net,
-               gw, lo, hi)
-        try:
-            self.run(cmds)
-        except CommandError, e:
-            log ("ignore: ")
-            e.dump()
-                
-    def del_route(self, net, gw, lo, hi):
-        cmds =  """
-  ignore_errors
-  network %s
-  del_route %s %s %s
-  quit  """ % (net, gw, lo, hi)
-        self.run(cmds)
-
-    # add a route to a host
-    def add_route_host(self, net, uuid, gw, tgt):
-        self.add_uuid(net, uuid, tgt)
-        cmds =  """
-  network %s
-  add_route %s %s
-  quit """ % (net,
-              gw, tgt)
-        try:
-            self.run(cmds)
-        except CommandError, e:
-            log ("ignore: ")
-            e.dump()
-
-    # add a route to a range
-    def del_route_host(self, net, uuid, gw, tgt):
-        self.del_uuid(uuid)
-        cmds =  """
-  ignore_errors
-  network %s
-  del_route %s %s
-  quit  """ % (net, gw, tgt)
-        self.run(cmds)
-
  
  
-    def del_peer(self, net_type, nid, hostaddr):
-        if net_type  in ('tcp',) and not config.lctl_dump:
-                cmds =  """
-  ignore_errors
-  network %s
-  del_peer %s %s single_share
-  quit""" % (net_type,
-             nid, hostaddr)
-                self.run(cmds)
-        elif net_type  in ('openib','iib','vib','ra') and not config.lctl_dump:
-                cmds =  """
-  ignore_errors
-  network %s
-  del_peer %s single_share
-  quit""" % (net_type,
-             nid)
-                self.run(cmds)
-        
      # disconnect one connection
      def disconnect(self, srv):
      # disconnect one connection
      def disconnect(self, srv):
+        if not srv.nid_uuid:
+            panic('nid_uuid not set for ', srv.net_type, srv.nid)
          self.del_uuid(srv.nid_uuid)
          self.del_uuid(srv.nid_uuid)
-        if srv.net_type  in ('tcp','openib','iib','vib','ra') and not config.lctl_dump:
-            if srv.hostaddr[0]:
-                hostaddr = string.split(srv.hostaddr[0], '/')[0]
-            self.del_peer(srv.net_type, srv.nid, hostaddr)
  
      def del_uuid(self, uuid):
          cmds =  """
  
      def del_uuid(self, uuid):
          cmds =  """
@@ -572,28 +397,27 @@ class LCTLInterface:
    quit""" % (uuid,)
          self.run(cmds)
  
    quit""" % (uuid,)
          self.run(cmds)
  
-    # disconnect all
-    def disconnectAll(self, net):
-        cmds =  """
-  ignore_errors
-  network %s
-  disconnect
-  quit""" % (net)
-        self.run(cmds)
-
      def attach(self, type, name, uuid):
          cmds = """
    attach %s %s %s
    quit""" % (type, name, uuid)
          self.run(cmds)
      def attach(self, type, name, uuid):
          cmds = """
    attach %s %s %s
    quit""" % (type, name, uuid)
          self.run(cmds)
-        
+
      def setup(self, name, setup = ""):
          cmds = """
    cfg_device %s
    setup %s
    quit""" % (name, setup)
          self.run(cmds)
      def setup(self, name, setup = ""):
          cmds = """
    cfg_device %s
    setup %s
    quit""" % (name, setup)
          self.run(cmds)
-        
+
+    def abort_recovery(self, name):
+        cmds = """
+  ignore_errors
+  device $%s
+  abort_recovery
+  quit""" % (name)
+        self.run(cmds)
+
      def add_conn(self, name, conn_uuid):
          cmds = """
    cfg_device %s
      def add_conn(self, name, conn_uuid):
          cmds = """
    cfg_device %s
@@ -601,7 +425,6 @@ class LCTLInterface:
    quit""" % (name, conn_uuid)
          self.run(cmds)
  
    quit""" % (name, conn_uuid)
          self.run(cmds)
  
-
      # create a new device with lctl
      def newdev(self, type, name, uuid, setup = ""):
          self.attach(type, name, uuid);
      # create a new device with lctl
      def newdev(self, type, name, uuid, setup = ""):
          self.attach(type, name, uuid);
@@ -610,7 +433,9 @@ class LCTLInterface:
          except CommandError, e:
              self.cleanup(name, uuid, 0)
              raise e
          except CommandError, e:
              self.cleanup(name, uuid, 0)
              raise e
-        
+        if (config.abort_recovery):
+            if (type == 'obdfilter' or type == 'mds'):
+                self.abort_recovery(name)
  
      # cleanup a device
      def cleanup(self, name, uuid, force, failover = 0):
  
      # cleanup a device
      def cleanup(self, name, uuid, force, failover = 0):
@@ -625,41 +450,34 @@ class LCTLInterface:
          self.run(cmds)
  
      # create an lov
          self.run(cmds)
  
      # create an lov
-    def lov_setup(self, name, uuid, desc_uuid, stripe_cnt,
-                  stripe_sz, stripe_off, pattern, devlist = None):
+    def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt,
+                  stripe_sz, stripe_off, pattern):
          cmds = """
    attach lov %s %s
          cmds = """
    attach lov %s %s
-  lov_setup %s %d %d %d %s %s
-  quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off,
-             pattern, devlist)
+  lov_setup %s %d %d %d %s
+  quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern)
          self.run(cmds)
  
      # add an OBD to a LOV
      def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
          cmds = """
          self.run(cmds)
  
      # add an OBD to a LOV
      def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
          cmds = """
+  cfg_device %s
    lov_modify_tgts add %s %s %s %s
    lov_modify_tgts add %s %s %s %s
-  quit""" % (name, obd_uuid, index, gen)
-        self.run(cmds)
-
-    # create an lmv
-    def lmv_setup(self, name, uuid, desc_uuid, devlist):
-        cmds = """
-  attach lmv %s %s
-  lmv_setup %s %s
-  quit""" % (name, uuid, desc_uuid, devlist)
+  quit""" % (name, name, obd_uuid, index, gen)
          self.run(cmds)
  
      # delete an OBD from a LOV
      def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
          cmds = """
          self.run(cmds)
  
      # delete an OBD from a LOV
      def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
          cmds = """
+  cfg_device %s
    lov_modify_tgts del %s %s %s %s
    lov_modify_tgts del %s %s %s %s
-  quit""" % (name, obd_uuid, index, gen)
+  quit""" % (name, name, obd_uuid, index, gen)
          self.run(cmds)
  
      # deactivate an OBD
      def deactivate(self, name):
          cmds = """
          self.run(cmds)
  
      # deactivate an OBD
      def deactivate(self, name):
          cmds = """
-  device $%s
+  cfg_device %s
    deactivate
    quit""" % (name)
          self.run(cmds)
    deactivate
    quit""" % (name)
          self.run(cmds)
@@ -673,15 +491,25 @@ class LCTLInterface:
  
      # get list of devices
      def device_list(self):
  
      # get list of devices
      def device_list(self):
-        devices = '/proc/fs/lustre/devices'
          ret = []
          ret = []
-        if os.access(devices, os.R_OK):
-            try:
-                fp = open(devices, 'r')
-                ret =  fp.readlines()
-                fp.close()
-            except IOError, e:
-                log(e)
+        if PLATFORM == 'LINUX':
+            devices = '/proc/fs/lustre/devices'
+            if os.access(devices, os.R_OK):
+                try:
+                    fp = open(devices, 'r')
+                    ret =  fp.readlines()
+                    fp.close()
+                except IOError, e:
+                    log(e)
+        elif PLATFORM == 'DARWIN':
+            rc, out = self.run("device_list")
+            ret = out.split("\n")
+            if len(ret) == 0:
+                return ret
+            tail = ret[-1]
+            if not tail:
+                # remove the last empty line
+                ret = ret[:-1]
          return ret
  
      # get lustre version
          return ret
  
      # get lustre version
@@ -709,6 +537,7 @@ class LCTLInterface:
    quit""" % (timeout,)
          self.run(cmds)
  
    quit""" % (timeout,)
          self.run(cmds)
  
+    # set lustre upcall
      def set_lustre_upcall(self, upcall):
          cmds = """
    set_lustre_upcall %s
      def set_lustre_upcall(self, upcall):
          cmds = """
    set_lustre_upcall %s
@@ -776,6 +605,17 @@ def do_find_file(base, mod):
              if module:
                  return module
  
              if module:
                  return module
  
+def find_module(src_dir, dev_dir, modname):
+    modbase = src_dir +'/'+ dev_dir +'/'+ modname
+    for modext in '.ko', '.o':
+        module = modbase + modext
+        try:
+            if os.access(module, os.R_OK):
+                return module
+        except OSError:
+            pass
+    return None
+
  # is the path a block device?
  def is_block(path):
      s = ()
  # is the path a block device?
  def is_block(path):
      s = ()
@@ -785,18 +625,31 @@ def is_block(path):
          return 0
      return stat.S_ISBLK(s[stat.ST_MODE])
  
          return 0
      return stat.S_ISBLK(s[stat.ST_MODE])
  
-# find the journal device from mkfs options
-def jdev(opts):
-    if opts == None:
-        return ''
-    x=string.split(opts)
-    i=0
-    while i < len(x) - 1:
-        if x[i] == '-J' and x[i+1].startswith('device='):
-            str=x[i+1]
-            return str[7:]
-        i=i+1
-    return ''
+def my_realpath(path):
+    try:
+        if os.path.islink(path):
+            # get the realpath of the mount point path
+            if 'realpath' in dir(os.path):
+                real_path = os.path.realpath(path)
+            else:
+                real_path = path
+                link_count = 0
+                while os.path.islink(real_path) and (link_count < 20):
+                    link_count = link_count + 1
+                    path_link = os.readlink(real_path)
+                    if os.path.isabs(path_link):
+                        real_path = path_link
+                    else:
+                        real_path = os.path.join(os.path.dirname(real_path), path_link)
+                    if link_count > 19:
+                        panic("Encountered too many symbolic links resolving path:", path)
+        else:
+            real_path = path
+
+        return real_path
+    except:
+        panic("Fatal error realpath()ing path:", path)
+
  
  # build fs according to type
  # fixme: dangerous
  
  # build fs according to type
  # fixme: dangerous
@@ -811,10 +664,9 @@ def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
          # devsize is in 1k, and fs block count is in 4k
          block_cnt = devsize/4
  
          # devsize is in 1k, and fs block count is in 4k
          block_cnt = devsize/4
  
-    if fstype in ('ext3', 'extN', 'ldiskfs'):
+    if fstype in ('ext3', 'ldiskfs'):
          # ext3 journal size is in megabytes
          # ext3 journal size is in megabytes
-        # but don't set jsize if mkfsoptions indicates a separate journal device
-        if jsize == 0 and jdev(mkfsoptions) == '':
+        if jsize == 0:
              if devsize == 0:
                  if not is_block(dev):
                      ret, out = runcmd("ls -l %s" %dev)
              if devsize == 0:
                  if not is_block(dev):
                      ret, out = runcmd("ls -l %s" %dev)
@@ -831,23 +683,10 @@ def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
                          # get the realpath of the device
                          # it may be the real device, such as /dev/hda7
                          # or the hardlink created via mknod for a device
                          # get the realpath of the device
                          # it may be the real device, such as /dev/hda7
                          # or the hardlink created via mknod for a device
-                        if 'realpath' in dir(os.path):
-                            real_dev = os.path.realpath(dev)
-                        else:
-                            real_dev = dev
-                            link_count = 0
-                            while os.path.islink(real_dev) and (link_count < 20):
-                                link_count = link_count + 1
-                                dev_link = os.readlink(real_dev)
-                                if os.path.isabs(dev_link):
-                                    real_dev = dev_link
-                                else:
-                                    real_dev = os.path.join(os.path.dirname(real_dev), dev_link)
-                                if link_count > 19:
-                                    panic("Entountered too many symbolic links resolving block device:", dev)
+                        real_dev = my_realpath(dev)
  
                          # get the major and minor number of the realpath via ls
  
                          # get the major and minor number of the realpath via ls
-                        # it seems python(os.stat) does not return 
+                        # it seems python(os.stat) does not return
                          # the st_rdev member of the stat structure
                          ret, out = runcmd("ls -l %s" %real_dev)
                          major = string.split(string.split(out[0])[4], ",")[0]
                          # the st_rdev member of the stat structure
                          ret, out = runcmd("ls -l %s" %real_dev)
                          major = string.split(string.split(out[0])[4], ",")[0]
@@ -864,20 +703,12 @@ def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
              if devsize > 1024 * 1024:
                  jsize = ((devsize / 102400) * 4)
              if jsize > 400:
              if devsize > 1024 * 1024:
                  jsize = ((devsize / 102400) * 4)
              if jsize > 400:
-                jsize = 400        
+                jsize = 400
          if jsize:  jopt = "-J size=%d" %(jsize,)
          if isize:  iopt = "-I %d" %(isize,)
          mkfs = 'mkfs.ext2 -j -b 4096 '
          if not isblock or config.force:
              mkfs = mkfs + ' -F '
          if jsize:  jopt = "-J size=%d" %(jsize,)
          if isize:  iopt = "-I %d" %(isize,)
          mkfs = 'mkfs.ext2 -j -b 4096 '
          if not isblock or config.force:
              mkfs = mkfs + ' -F '
-        if jdev(mkfsoptions) != '':
-            jmkfs = 'mkfs.ext2 -b 4096 -O journal_dev '
-            if config.force:
-                jmkfs = jmkfs + '-F '
-            jmkfs = jmkfs + jdev(mkfsoptions)                
-            (ret, out) = run (jmkfs)
-            if ret:
-                panic("Unable format journal device:", jdev(mkfsoptions), string.join(out))
      elif fstype == 'reiserfs':
          # reiserfs journal size is in blocks
          if jsize:  jopt = "--journal_size %d" %(jsize,)
      elif fstype == 'reiserfs':
          # reiserfs journal size is in blocks
          if jsize:  jopt = "--journal_size %d" %(jsize,)
@@ -893,8 +724,8 @@ def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
      if ret:
          panic("Unable to build fs:", dev, string.join(out))
      # enable hash tree indexing on fsswe
      if ret:
          panic("Unable to build fs:", dev, string.join(out))
      # enable hash tree indexing on fsswe
-    if fstype in ('ext3', 'extN', 'ldiskfs'):
-        htree = 'echo "feature FEATURE_C5" | debugfs -w'
+    if fstype in ('ext3', 'ldiskfs'):
+        htree = 'tune2fs -O dir_index'
          (ret, out) = run (htree, dev)
          if ret:
              panic("Unable to enable htree:", dev)
          (ret, out) = run (htree, dev)
          if ret:
              panic("Unable to enable htree:", dev)
@@ -906,11 +737,11 @@ def loop_base():
      if not os.access(loop + str(0), os.R_OK):
          loop = loop + '/'
          if not os.access(loop + str(0), os.R_OK):
      if not os.access(loop + str(0), os.R_OK):
          loop = loop + '/'
          if not os.access(loop + str(0), os.R_OK):
-            panic ("can't access loop devices")
+            loop='/dev/loop'
      return loop
      return loop
-    
+
  # find loop device assigned to the file
  # find loop device assigned to the file
-def find_assigned_loop(file):
+def find_loop(file):
      loop = loop_base()
      for n in xrange(0, MAX_LOOP_DEVICES):
          dev = loop + str(n)
      loop = loop_base()
      for n in xrange(0, MAX_LOOP_DEVICES):
          dev = loop + str(n)
@@ -920,90 +751,63 @@ def find_assigned_loop(file):
                  m = re.search(r'\((.*)\)', out[0])
                  if m and file == m.group(1):
                      return dev
                  m = re.search(r'\((.*)\)', out[0])
                  if m and file == m.group(1):
                      return dev
+        else:
+            break
      return ''
  
      return ''
  
-# find free loop device
-def find_free_loop(file):
+# create file if necessary and assign the first free loop device
+def init_loop(file, size, fstype, journal_size, inode_size, mkfsoptions, reformat):
+    dev = find_loop(file)
+    if dev:
+        print 'WARNING file:', file, 'already mapped to', dev
+        return dev
+    if reformat or not os.access(file, os.R_OK | os.W_OK):
+        if size < 8000:
+            panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (file,size))
+        (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size,
+                                                                         file))
+        if ret:
+            panic("Unable to create backing store:", file)
+        mkfs(file, size, fstype, journal_size, inode_size, mkfsoptions, isblock=0)
+
      loop = loop_base()
      loop = loop_base()
-    
      # find next free loop
      for n in xrange(0, MAX_LOOP_DEVICES):
          dev = loop + str(n)
          if os.access(dev, os.R_OK):
              (stat, out) = run('losetup', dev)
              if stat:
      # find next free loop
      for n in xrange(0, MAX_LOOP_DEVICES):
          dev = loop + str(n)
          if os.access(dev, os.R_OK):
              (stat, out) = run('losetup', dev)
              if stat:
+                (stat, out) = run('losetup', dev, file)
+                if stat:
+                    panic("losetup failed: (%s) %s" % (stat, out[0].strip()))
                  return dev
                  return dev
+        else:
+            print "out of loop devices"
+            return ''
+    print "out of loop devices"
      return ''
  
      return ''
  
-# create file if necessary and assign the first free loop device
-def init_loop(file, size, fstype, journal_size, inode_size, 
-              mkfsoptions, reformat, autoformat, backfstype, backfile):
-    if fstype == 'smfs':
-        realfile = backfile
-        realfstype = backfstype
-        if is_block(backfile):
-            if reformat or (need_format(realfstype, backfile) and autoformat == 'yes'):
-                mkfs(realfile, size, realfstype, journal_size, inode_size, mkfsoptions, isblock=0)
-            return realfile
-    else:
-        realfile = file
-        realfstype = fstype
-            
-    dev = find_assigned_loop(realfile)
+# undo loop assignment
+def clean_loop(file):
+    dev = find_loop(file)
      if dev:
      if dev:
-        print 'WARNING: file', realfile, 'already mapped to', dev
-       return dev
-            
-    if reformat or not os.access(realfile, os.R_OK | os.W_OK):
-        (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, realfile))
+        ret, out = run('losetup -d', dev)
          if ret:
          if ret:
-            panic("Unable to create backing store:", realfile)
-        mkfs(realfile, size, realfstype, journal_size, inode_size, 
-             mkfsoptions, isblock=0)
-
-    dev = find_free_loop(realfile)
-    if dev:
-       print "attach " + realfile + " <-> " + dev                  
-        run('losetup', dev, realfile)
-        return dev
-
-    print "out of loop devices"
-    return ''
+            log('unable to clean loop device:', dev, 'for file:', file)
+            logall(out)
  
  
-# undo loop assignment
-def clean_loop(dev, fstype, backfstype, backdev):
-    if fstype == 'smfs':
-       realfile = backdev
-    else:
-       realfile = dev
-    if not is_block(realfile):
-       dev = find_assigned_loop(realfile)
-       if dev:
-           print "detach " + dev + " <-> " + realfile
-           ret, out = run('losetup -d', dev)
-           if ret:
-               log('unable to clean loop device:', dev, 'for file:', realfile)
-               logall(out)
-
-# finilizes passed device
-def clean_dev(dev, fstype, backfstype, backdev):
-    if fstype == 'smfs' or not is_block(dev):
-       clean_loop(dev, fstype, backfstype, backdev)
-       
  # determine if dev is formatted as a <fstype> filesystem
  def need_format(fstype, dev):
  # determine if dev is formatted as a <fstype> filesystem
  def need_format(fstype, dev):
-    # FIXME don't know how to implement this    
+    # FIXME don't know how to implement this
      return 0
  
  # initialize a block device if needed
  def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
      return 0
  
  # initialize a block device if needed
  def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
-              inode_size, mkfsoptions, backfstype, backdev):
-    if config.noexec: 
-        return dev
-        
-    if fstype == 'smfs' or not is_block(dev):
+              inode_size, mkfsoptions):
+    if config.noexec: return dev
+    if not is_block(dev):
          dev = init_loop(dev, size, fstype, journal_size, inode_size,
          dev = init_loop(dev, size, fstype, journal_size, inode_size,
-                        mkfsoptions, reformat, autoformat, backfstype, backdev)
+                        mkfsoptions, reformat)
      elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
          mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
               isblock=0)
      elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
          mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
               isblock=0)
@@ -1011,7 +815,6 @@ def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
  #        panic("device:", dev,
  #              "not prepared, and autoformat is not set.\n",
  #              "Rerun with --reformat option to format ALL filesystems")
  #        panic("device:", dev,
  #              "not prepared, and autoformat is not set.\n",
  #              "Rerun with --reformat option to format ALL filesystems")
-        
      return dev
  
  def if2addr(iface):
      return dev
  
  def if2addr(iface):
@@ -1023,87 +826,106 @@ def if2addr(iface):
      ip = string.split(addr, ':')[1]
      return ip
  
      ip = string.split(addr, ':')[1]
      return ip
  
-def def_mount_options(fstype, target):
+def def_mount_options(fstype, target, blkdev):
      """returns deafult mount options for passed fstype and target (mds, ost)"""
      if fstype == 'ext3' or fstype == 'ldiskfs':
          mountfsoptions = "errors=remount-ro"
      """returns deafult mount options for passed fstype and target (mds, ost)"""
      if fstype == 'ext3' or fstype == 'ldiskfs':
          mountfsoptions = "errors=remount-ro"
-        if target == 'ost' and sys_get_branch() == '2.4':
-            mountfsoptions = "%s,asyncdel" % (mountfsoptions)
-        return mountfsoptions
-    return ""
+        if target == 'ost':
+            if sys_get_branch() == '2.4':
+                mountfsoptions = "%s,asyncdel" % (mountfsoptions)
+            #else:
+            #    mountfsoptions = "%s,extents,mballoc" % (mountfsoptions)
+        elif target == 'mds':
+            if config.user_xattr:
+                mountfsoptions = "%s,user_xattr" % (mountfsoptions)
+            if config.acl:
+                mountfsoptions = "%s,acl" % (mountfsoptions)
+
+        if blkdev:
+            # grab superblock info
+            dumpe2fs="dumpe2fs -f -h"
+            (ret, sb) = run(dumpe2fs, blkdev)
+            if ret:
+                panic("unable to get superblock for ", blkdev)
+
+            # extract journal UUID
+            journal_UUID=''
+            journal_DEV=''
+            for line in sb:
+                lst = string.split(line, ":")
+                if lst[0] == 'Journal UUID':
+                    if len(lst[1]) < 3:
+                        panic("cannot retrieve journal UUID for ", blkdev)
+                    if string.split(lst[1])[0] != '<none>':
+                        journal_UUID = string.split(lst[1])[0]
+                        debug(blkdev, 'has journal UUID', journal_UUID)
+                if lst[0] == 'Journal device':
+                    if len(lst[1]) < 3:
+                        panic("cannot retrieve journal device for ", blkdev)
+                    if string.split(lst[1])[0] != '0x0000':
+                        journal_DEV = string.split(lst[1])[0]
+                        debug(blkdev, 'has journal device', journal_DEV)
+                    break
+
+            if len(journal_UUID) == 0 or len(journal_DEV) == 0:
+                debug('no external journal found for', blkdev)
+                # use internal journal
+                return mountfsoptions
          
          
-def sys_get_elan_position_file():
-    procfiles = ["/proc/elan/device0/position",
-                 "/proc/qsnet/elan4/device0/position",
-                 "/proc/qsnet/elan3/device0/position"]
-    for p in procfiles:
-        if os.access(p, os.R_OK):
-            return p
+            # run blkid, lookup highest-priority device with matching UUID
+            blkid = "blkid -o device -l -t UUID='%s'" % (journal_UUID)
+            (ret, devname) = run(blkid)
+            if ret or len(devname) == 0:
+                panic("cannot find external journal for ", blkdev)
+            debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
+                  string.replace(devname[0], '\n', ''))
+
+            try: # sigh, python 1.5 does not support os.stat().st_rdev
+                jdevpath = my_realpath(string.replace(devname[0], '\n', ''))
+                ret, out = runcmd("ls -l %s" %jdevpath)
+                debug('ls -l:', out)
+                major = int(string.split(string.split(out[0])[4], ',')[0])
+                minor = int(string.split(out[0])[5])
+                debug('major', major, 'minor', minor)
+                rdev = major << 8 | minor
+            except OSError:
+                panic("cannot stat ", devname[0])
+
+            debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
+                  jdevpath, 'rdev', rdev)
+
+            # add mount option
+            if string.atoi(journal_DEV, 0) != rdev:
+                mountfsoptions =  "%s,journal_dev=%#x" % (mountfsoptions,rdev)
+
+        return mountfsoptions
      return ""
  
      return ""
  
-def sys_get_local_nid(net_type, wildcard, cluster_id):
-    """Return the local nid."""
-    local = ""
-    if sys_get_elan_position_file():
-        local = sys_get_local_address('elan', '*', cluster_id)
-    else:
-        local = sys_get_local_address(net_type, wildcard, cluster_id)
-    return local
-        
-def sys_get_local_address(net_type, wildcard, cluster_id):
-    """Return the local address for the network type."""
-    local = ""
-    if net_type in ('tcp','openib','iib','vib','ra'):
-        if  ':' in wildcard:
-            iface, star = string.split(wildcard, ':')
-            local = if2addr(iface)
-            if not local:
-                panic ("unable to determine ip for:", wildcard)
-        else:
-            host = socket.gethostname()
-            local = socket.gethostbyname(host)
-    elif net_type == 'elan':
-        # awk '/NodeId/ { print $2 }' 'sys_get_elan_position_file()'
-        f = sys_get_elan_position_file()
-        if not f:
-            panic ("unable to determine local Elan ID")
+def sys_get_branch():
+    """Returns kernel release"""
+    return os.uname()[2][:3]
+
+def mod_loaded(modname):
+    """Check if a module is already loaded. Look in /proc/modules for it."""
+    if PLATFORM == 'LINUX':
          try:
          try:
-            fp = open(f, 'r')
+            fp = open('/proc/modules')
              lines = fp.readlines()
              fp.close()
              lines = fp.readlines()
              fp.close()
-            for l in lines:
-                a = string.split(l)
-                if a[0] == 'NodeId':
-                    elan_id = a[1]
-                    break
-            try:
-                nid = my_int(cluster_id) + my_int(elan_id) 
-                local = "%d" % (nid)
-            except ValueError, e:
-                local = elan_id
-        except IOError, e:
-            log(e)
-    elif net_type == 'lo':
-        fixme("automatic local address for loopback")
-    elif net_type == 'gm':
-        fixme("automatic local address for GM")
-
-    return local
-
-def sys_get_branch():
-    """Returns kernel release"""
-    try:
-        fp = open('/proc/sys/kernel/osrelease')
-        lines = fp.readlines()
-        fp.close()
-        
-        for l in lines:
-            version = string.split(l)
-            a = string.split(version[0], '.')
-            return a[0] + '.' + a[1]
-    except IOError, e:
-        log(e)
-    return ""
+            # please forgive my tired fingers for this one
+            ret = filter(lambda word, mod=modname: word == mod,
+                         map(lambda line: string.split(line)[0], lines))
+            return ret
+        except Exception, e:
+            return 0
+    elif PLATFORM == 'DARWIN':
+        ret, out = run('/usr/sbin/kextstat | /usr/bin/grep', modname)
+        if ret == 0:
+                return 1
+        else:
+                return 0
+    else:
+        return 0
  
  # XXX: instead of device_list, ask for $name and see what we get
  def is_prepared(name):
  
  # XXX: instead of device_list, ask for $name and see what we get
  def is_prepared(name):
@@ -1123,7 +945,7 @@ def is_prepared(name):
          e.dump()
      return 0
  
          e.dump()
      return 0
  
-def net_is_prepared():
+def is_network_prepared():
      """If the any device exists, then assume that all networking
         has been configured"""
      out = lctl.device_list()
      """If the any device exists, then assume that all networking
         has been configured"""
      out = lctl.device_list()
@@ -1132,165 +954,119 @@ def net_is_prepared():
  def fs_is_mounted(path):
      """Return true if path is a mounted lustre filesystem"""
      try:
  def fs_is_mounted(path):
      """Return true if path is a mounted lustre filesystem"""
      try:
+        real_path = my_realpath(path)
+
          fp = open('/proc/mounts')
          lines = fp.readlines()
          fp.close()
          for l in lines:
              a = string.split(l)
          fp = open('/proc/mounts')
          lines = fp.readlines()
          fp.close()
          for l in lines:
              a = string.split(l)
-            if a[1] == path and a[2] == 'lustre_lite':
+            if a[1] == real_path and a[2] == 'lustre_lite':
                  return 1
      except IOError, e:
          log(e)
      return 0
  
                  return 1
      except IOError, e:
          log(e)
      return 0
  
-def kmod_find(src_dir, dev_dir, modname):
-    modbase = src_dir +'/'+ dev_dir +'/'+ modname
-    for modext in '.ko', '.o':
-        module = modbase + modext
-        try:
-            if os.access(module, os.R_OK):
-                return module
-        except OSError:
-               pass
-    return None
-
-def kmod_info(modname):
-    """Returns reference count for passed module name."""
-    try:
-       fp = open('/proc/modules')
-       lines = fp.readlines()
-       fp.close()
-       
-       # please forgive my tired fingers for this one
-       ret = filter(lambda word, mod = modname: word[0] == mod,
-                    map(lambda line: string.split(line), lines))
-       if not ret:
-           return ''
-       return ret[0]
-    except Exception, e:
-        return 0
-
  class kmod:
  class kmod:
-    """Presents kernel module"""
-    def __init__(self, src_dir, dev_dir, name):
-        self.src_dir = src_dir
-        self.dev_dir = dev_dir
-        self.name = name
-
-    def load(self):
-        """Load module"""
-        log ('loading module:', self.name, 'srcdir',
-             self.src_dir, 'devdir', self.dev_dir)
-        if self.src_dir:
-            module = kmod_find(self.src_dir, self.dev_dir,
-                               self.name)
-            if not module:
-                panic('module not found:', self.name)
-            (rc, out)  = run('/sbin/insmod', module)
-            if rc:
-                raise CommandError('insmod', out, rc)
-        else:
-            (rc, out) = run('/sbin/modprobe', self.name)
-            if rc:
-                raise CommandError('modprobe', out, rc)
-
-    def cleanup(self):
-       """Unload module"""
-        log('unloading module:', self.name)
-        (rc, out) = run('/sbin/rmmod', self.name)
-        if rc:
-            log('unable to unload module:', self.name +
-                "(" + self.refcount() + ")")
-            logall(out)
-
-    def info(self):
-        """Returns module info if any."""
-        return kmod_info(self.name)
-
-    def loaded(self):
-        """Returns 1 if module is loaded. Otherwise 0 is returned."""
-        if self.info():
-            return 1
-        else:
-            return 0
-
-    def refcount(self):
-        """Returns module refcount."""
-        info = self.info()
-        if not info:
-            return ''
-        return info[2]
-
-    def used(self):
-        """Returns 1 if module is used, otherwise 0 is returned."""
-        info = self.info()
-        if not info:
-            return 0
-        if len(info) > 3:
-            users = info[3]
-            if users and users != '(unused)' and users != '-':
-                return 1
-            else:
-                return 0
-        else:
-            return 0
-
-    def busy(self):
-        """Returns 1 if module is busy, otherwise 0 is returned."""
-        if self.loaded() and (self.used() or self.refcount() != '0'):
-            return 1
-        else:
-            return 0
-
-class kmod_manager:
      """Manage kernel modules"""
      def __init__(self, lustre_dir, portals_dir):
          self.lustre_dir = lustre_dir
          self.portals_dir = portals_dir
          self.kmodule_list = []
  
      """Manage kernel modules"""
      def __init__(self, lustre_dir, portals_dir):
          self.lustre_dir = lustre_dir
          self.portals_dir = portals_dir
          self.kmodule_list = []
  
-    def find_module(self, modname):
-        """Find module by module name"""
-        for mod in self.kmodule_list:
-            if mod.name == modname:
-                return mod
-        return ''
-        
      def add_portals_module(self, dev_dir, modname):
          """Append a module to list of modules to load."""
      def add_portals_module(self, dev_dir, modname):
          """Append a module to list of modules to load."""
-
-        mod = self.find_module(modname)
-        if not mod:
-            mod = kmod(self.portals_dir, dev_dir, modname)
-            self.kmodule_list.append(mod)
+        self.kmodule_list.append((self.portals_dir, dev_dir, modname))
  
      def add_lustre_module(self, dev_dir, modname):
          """Append a module to list of modules to load."""
  
      def add_lustre_module(self, dev_dir, modname):
          """Append a module to list of modules to load."""
+        self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
  
  
-        mod = self.find_module(modname)
-        if not mod:
-            mod = kmod(self.lustre_dir, dev_dir, modname)
-            self.kmodule_list.append(mod)
-        
-    def load_modules(self):
+    def load_module(self):
          """Load all the modules in the list in the order they appear."""
          """Load all the modules in the list in the order they appear."""
-        for mod in self.kmodule_list:
-            if mod.loaded() and not config.noexec:
+        for src_dir, dev_dir, mod in self.kmodule_list:
+            if mod_loaded(mod) and not config.noexec:
                  continue
                  continue
-            mod.load()
-
-    def cleanup_modules(self):
+            log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
+            if PLATFORM == 'LINUX':
+                options = ''
+                if mod == 'lnet':
+                    #For LNET we really need modprobe to load defined LNDs
+                    run('/sbin/modprobe lnet')
+                    #But if that fails, try insmod anyhow with dev option
+                    #accept=all for dev liblustre testing
+                    options = 'accept=all'
+                if src_dir:
+                    module = find_module(src_dir, dev_dir, mod)
+                    if not module:
+                        panic('module not found:', mod)
+                    (rc, out) = run('/sbin/insmod', module, options)
+                    if rc and not mod_loaded(mod):
+                        if rc == 1:
+                            print("Bad module options? Check dmesg.") 
+                        raise CommandError('insmod', out, rc)
+                else:
+                    (rc, out) = run('/sbin/modprobe', mod)
+                    if rc and not mod_loaded(mod):
+                        if rc == 1:
+                            print("Bad module options? Check dmesg.") 
+                        raise CommandError('modprobe', out, rc)
+            elif PLATFORM == 'DARWIN':
+                run('/sbin/kextload', KEXTPATH + mod + '.kext');
+
+    def cleanup_module(self):
          """Unload the modules in the list in reverse order."""
          """Unload the modules in the list in reverse order."""
-        rev = self.kmodule_list
+
+        rev = self.kmodule_list[:] # make *copy* of list
          rev.reverse()
          rev.reverse()
-        for mod in rev:
-            if (not mod.loaded() or mod.busy()) and not config.noexec:
+        for src_dir, dev_dir, mod in rev:
+            if not mod_loaded(mod) and not config.noexec:
                  continue
                  continue
-            # debug hack
-            if mod.name == 'portals' and config.dump:
-                lctl.dump(config.dump)
-            mod.cleanup()
-           
+            if mod == 'ksocklnd' and not config.noexec:
+                # Ignore ksocklnd in module list (lnet will remove)
+                continue
+            log('unloading module:', mod)
+            if mod == 'lnet' and not config.noexec:
+                # remove any self-ref portals created
+                lctl.unconfigure_network()
+                if config.dump:
+                    debug('dumping debug log to', config.dump)
+                    # debug hack
+                    lctl.dump(config.dump)
+                log('unloading the network')
+                lctl.unconfigure_network()
+                if mod_loaded("ksocklnd"):
+                    if PLATFORM == 'LINUX':
+                        run('/sbin/rmmod ksocklnd')
+                    elif PLATFORM == 'DARWIN':
+                        run('/sbin/kextunload', KEXTPATH+'ksocklnd.kext')
+                if mod_loaded("kqswlnd"):
+                    run('/sbin/rmmod kqswlnd')
+                if mod_loaded("kgmlnd"):
+                    run('/sbin/rmmod kgmlnd')
+                if mod_loaded("kopeniblnd"):
+                    run('/sbin/rmmod kopeniblnd')
+                if mod_loaded("kiiblnd"):
+                    run('/sbin/rmmod kiiblnd')
+                if mod_loaded("kviblnd"):
+                    run('/sbin/rmmod kviblnd')
+                if mod_loaded("kciblnd"):
+                    run('/sbin/rmmod kciblnd')
+                if mod_loaded("ko2iblnd"):
+                    run('/sbin/rmmod ko2iblnd')
+                if mod_loaded("kralnd"):
+                    run('/sbin/rmmod kralnd')
+                if mod_loaded("kptllnd"):
+                    run('/sbin/rmmod kptllnd')
+            if PLATFORM == 'LINUX':
+                (rc, out) = run('/sbin/rmmod', mod)
+            elif PLATFORM == 'DARWIN':
+                (rc, out) = run('/sbin/kextunload', KEXTPATH+mod+'.kext');
+            if rc:
+                log('! unable to unload module:', mod)
+                logall(out)
+
+
  # ============================================================
  # Classes to prepare and cleanup the various objects
  #
  # ============================================================
  # Classes to prepare and cleanup the various objects
  #
@@ -1305,10 +1081,11 @@ class Module:
          self.uuid = self.db.getUUID()
          self._server = None
          self._connected = 0
          self.uuid = self.db.getUUID()
          self._server = None
          self._connected = 0
+        self.kmod = kmod(config.lustre, config.portals)
  
      def info(self, *args):
          msg = string.join(map(str,args))
  
      def info(self, *args):
          msg = string.join(map(str,args))
-        print self.module_name + ":", self.name, self.uuid, msg
+        log (self.module_name + ":", self.name, self.uuid, msg)
  
      def cleanup(self):
          """ default cleanup, used for most modules """
  
      def cleanup(self):
          """ default cleanup, used for most modules """
@@ -1320,232 +1097,71 @@ class Module:
              e.dump()
              cleanup_error(e.rc)
  
              e.dump()
              cleanup_error(e.rc)
  
-    def add_module(self, manager):
-        """Adds all needed modules in the order they appear."""
-        return
+    def add_portals_module(self, dev_dir, modname):
+        """Append a module to list of modules to load."""
+        self.kmod.add_portals_module(dev_dir, modname)
+
+    def add_lustre_module(self, dev_dir, modname):
+        """Append a module to list of modules to load."""
+        self.kmod.add_lustre_module(dev_dir, modname)
+
+    def load_module(self):
+        """Load all the modules in the list in the order they appear."""
+        self.kmod.load_module()
+
+    def cleanup_module(self):
+        """Unload the modules in the list in reverse order."""
+        if self.safe_to_clean():
+            self.kmod.cleanup_module()
  
      def safe_to_clean(self):
          return 1
  
      def safe_to_clean_modules(self):
          return self.safe_to_clean()
  
      def safe_to_clean(self):
          return 1
  
      def safe_to_clean_modules(self):
          return self.safe_to_clean()
-        
+
  class Network(Module):
  class Network(Module):
-    def __init__(self,db):
+    def __init__(self,db,nid_uuid=0):
          Module.__init__(self, 'NETWORK', db)
          self.net_type = self.db.get_val('nettype')
          self.nid = self.db.get_val('nid', '*')
          self.cluster_id = self.db.get_val('clusterid', "0")
          self.port = self.db.get_val_int('port', 0)
          Module.__init__(self, 'NETWORK', db)
          self.net_type = self.db.get_val('nettype')
          self.nid = self.db.get_val('nid', '*')
          self.cluster_id = self.db.get_val('clusterid', "0")
          self.port = self.db.get_val_int('port', 0)
-
-        if '*' in self.nid:
-            self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id)
-            if not self.nid:
-                panic("unable to set nid for", self.net_type, self.nid, cluster_id)
-            self.generic_nid = 1
-            debug("nid:", self.nid)
-        else:
-            self.generic_nid = 0
-
-        self.nid_uuid = self.nid_to_uuid(self.nid)
-        self.hostaddr = self.db.get_hostaddr()
-        if len(self.hostaddr) == 0:
-            self.hostaddr.append(self.nid)
-        if '*' in self.hostaddr[0]:
-            self.hostaddr[0] = sys_get_local_address(self.net_type, self.hostaddr[0], self.cluster_id)
-            if not self.hostaddr[0]:
-                panic("unable to set hostaddr for", self.net_type, self.hostaddr[0], self.cluster_id)
-            debug("hostaddr:", self.hostaddr[0])
-
-    def add_module(self, manager):
-        manager.add_portals_module("libcfs", 'libcfs')
-        manager.add_portals_module("portals", 'portals')
-        if node_needs_router():
-            manager.add_portals_module("router", 'kptlrouter')
-        if self.net_type == 'tcp':
-            manager.add_portals_module("knals/socknal", 'ksocknal')
-        if self.net_type == 'elan':
-            manager.add_portals_module("knals/qswnal", 'kqswnal')
-        if self.net_type == 'gm':
-            manager.add_portals_module("knals/gmnal", 'kgmnal')
-        if self.net_type == 'openib':
-            manager.add_portals_module("knals/openibnal", 'kopenibnal')
-        if self.net_type == 'iib':
-            manager.add_portals_module("knals/iibnal", 'kiibnal')
-        if self.net_type == 'vib':
-            self.add_portals_module("knals/vibnal", 'kvibnal')
-        if self.net_type == 'lo':
-            manager.add_portals_module("knals/lonal", 'klonal')
-        if self.net_type == 'ra':
-            manager.add_portals_module("knals/ranal", 'kranal')
-
-    def nid_to_uuid(self, nid):
-        return "NID_%s_UUID" %(nid,)
+        self.nid_uuid = nid_uuid
+        self.add_portals_module('libcfs', 'libcfs')
+        self.add_portals_module('lnet', 'lnet')
+        # Add the socklnd for developers without modprobe.conf (umls)
+        self.add_portals_module('klnds/socklnd', 'ksocklnd')
  
      def prepare(self):
  
      def prepare(self):
-        if not config.record and net_is_prepared():
+        if is_network_prepared():
              return
              return
-        self.info(self.net_type, self.nid, self.port)
-        if not (config.record and self.generic_nid):
-            lctl.network(self.net_type, self.nid)
+        self.info(self.net_type, self.nid)
          if self.net_type == 'tcp':
              sys_tweak_socknal()
          if self.net_type == 'tcp':
              sys_tweak_socknal()
-            for hostaddr in self.db.get_hostaddr():
-                ip = string.split(hostaddr, '/')[0]
-                if len(string.split(hostaddr, '/')) == 2:
-                    netmask = string.split(hostaddr, '/')[1]
-                else:
-                    netmask = ""
-                lctl.add_interface(self.net_type, ip, netmask)
          if self.net_type == 'elan':
              sys_optimize_elan()
          if self.net_type == 'elan':
              sys_optimize_elan()
-        if self.port and  node_is_router():
-            run_one_acceptor(self.port)
-            self.connect_peer_gateways()
-
-    def connect_peer_gateways(self):
-        for router in self.db.lookup_class('node'):
-            if router.get_val_int('router', 0):
-                for netuuid in router.get_networks():
-                    net = self.db.lookup(netuuid)
-                    gw = Network(net)
-                    if (gw.cluster_id == self.cluster_id and
-                        gw.net_type == self.net_type):
-                        if gw.nid != self.nid:
-                            lctl.connect(gw)
-
-    def disconnect_peer_gateways(self):
-        for router in self.db.lookup_class('node'):
-            if router.get_val_int('router', 0):
-                for netuuid in router.get_networks():
-                    net = self.db.lookup(netuuid)
-                    gw = Network(net)
-                    if (gw.cluster_id == self.cluster_id and
-                        gw.net_type == self.net_type):
-                        if gw.nid != self.nid:
-                            try:
-                                lctl.disconnect(gw)
-                            except CommandError, e:
-                                print "disconnect failed: ", self.name
-                                e.dump()
-                                cleanup_error(e.rc)
-
-    def safe_to_clean(self):
-        return not net_is_prepared()
-
-    def cleanup(self):
-        self.info(self.net_type, self.nid, self.port)
-        if self.port:
-            stop_acceptor(self.port)
-        if  node_is_router():
-            self.disconnect_peer_gateways()
-        if self.net_type == 'tcp':
-            for hostaddr in self.db.get_hostaddr():
-                ip = string.split(hostaddr, '/')[0]
-                lctl.del_interface(self.net_type, ip)
-
-    def correct_level(self, level, op=None):
-        return level
-
-class RouteTable(Module):
-    def __init__(self,db):
-        Module.__init__(self, 'ROUTES', db)
-
-    def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id,
-                         lo, hi):
-        # only setup connections for tcp, openib, and iib NALs
-        srvdb = None
-        if not net_type in ('tcp','openib','iib','vib','ra'):
-            return None
-
-        # connect to target if route is to single node and this node is the gw
-        if lo == hi and local_interface(net_type, gw_cluster_id, gw):
-            if not local_cluster(net_type, tgt_cluster_id):
-                panic("target", lo, " not on the local cluster")
-            srvdb = self.db.nid2server(lo, net_type, gw_cluster_id)
-        # connect to gateway if this node is not the gw
-        elif (local_cluster(net_type, gw_cluster_id)
-              and not local_interface(net_type, gw_cluster_id, gw)):
-            srvdb = self.db.nid2server(gw, net_type, gw_cluster_id)
-        else:
-            return None
-
-        if not srvdb:
-            panic("no server for nid", lo)
-            return None
-
-        return Network(srvdb)
-        
-    def prepare(self):
-        if not config.record and net_is_prepared():
-            return
-        self.info()
-        for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
-            lctl.add_route(net_type, gw, lo, hi)
-            srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
-            if srv:
-                lctl.connect(srv)
-
-    def safe_to_clean(self):
-        return not net_is_prepared()
-
-    def cleanup(self):
-        if net_is_prepared():
-            # the network is still being used, don't clean it up
-            return
-        for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
-            srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
-            if srv:
-                try:
-                    lctl.disconnect(srv)
-                except CommandError, e:
-                    print "disconnect failed: ", self.name
-                    e.dump()
-                    cleanup_error(e.rc)
-
-            try:
-                lctl.del_route(net_type, gw, lo, hi)
-            except CommandError, e:
-                print "del_route failed: ", self.name
-                e.dump()
-                cleanup_error(e.rc)
-
-class Management(Module):
-    def __init__(self, db):
-        Module.__init__(self, 'MGMT', db)
-
-    def add_module(self, manager):
-        manager.add_lustre_module('lvfs', 'lvfs')
-        manager.add_lustre_module('obdclass', 'obdclass')
-        manager.add_lustre_module('ptlrpc', 'ptlrpc')
-        manager.add_lustre_module('mgmt', 'mgmt_svc')
-
-    def prepare(self):
-        if not config.record and is_prepared(self.name):
-            return
-        self.info()
-        lctl.newdev("mgmt", self.name, self.uuid)
  
      def safe_to_clean(self):
  
      def safe_to_clean(self):
+        if PLATFORM == 'LINUX':
+            return not is_network_prepared()
+        elif PLATFORM == 'DARWIN':
+            # XXX always assume it's safe to clean 
+            return 1    
          return 1
  
      def cleanup(self):
          return 1
  
      def cleanup(self):
-        if is_prepared(self.name):
-            Module.cleanup(self)
-
-    def correct_level(self, level, op=None):
-        return level
+        self.info(self.net_type, self.nid)
  
  # This is only needed to load the modules; the LDLM device
  # is now created automatically.
  class LDLM(Module):
      def __init__(self,db):
          Module.__init__(self, 'LDLM', db)
  
  # This is only needed to load the modules; the LDLM device
  # is now created automatically.
  class LDLM(Module):
      def __init__(self,db):
          Module.__init__(self, 'LDLM', db)
-
-    def add_module(self, manager):
-        manager.add_lustre_module('lvfs', 'lvfs')
-        manager.add_lustre_module('obdclass', 'obdclass')
-        manager.add_lustre_module('ptlrpc', 'ptlrpc')
+        self.add_lustre_module('lvfs', 'lvfs')
+        self.add_lustre_module('obdclass', 'obdclass')
+        self.add_lustre_module('ptlrpc', 'ptlrpc')
+        self.add_lustre_module('ptlrpc/gss', 'ptlrpc_gss')
  
      def prepare(self):
          return
  
      def prepare(self):
          return
@@ -1553,54 +1169,68 @@ class LDLM(Module):
      def cleanup(self):
          return
  
      def cleanup(self):
          return
  
-    def correct_level(self, level, op=None):
-        return level
-
  class LOV(Module):
      def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
          Module.__init__(self, 'LOV', db)
          if name_override != None:
              self.name = "lov_%s" % name_override
  class LOV(Module):
      def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
          Module.__init__(self, 'LOV', db)
          if name_override != None:
              self.name = "lov_%s" % name_override
+        self.add_lustre_module('lov', 'lov')
          self.mds_uuid = self.db.get_first_ref('mds')
          self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
          self.stripe_off = self.db.get_val_int('stripeoffset', 0)
          self.pattern = self.db.get_val_int('stripepattern', 0)
          self.mds_uuid = self.db.get_first_ref('mds')
          self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
          self.stripe_off = self.db.get_val_int('stripeoffset', 0)
          self.pattern = self.db.get_val_int('stripepattern', 0)
-        self.devlist = self.db.get_lov_tgts('lov_tgt')
-        self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist))
+        self.devlist = []
+        self.stripe_cnt = self.db.get_val_int('stripecount', 1)
          self.osclist = []
          self.osclist = []
-       self.obdlist = []
          self.desc_uuid = self.uuid
          self.uuid = generate_client_uuid(self.name)
          self.fs_name = fs_name
          self.desc_uuid = self.uuid
          self.uuid = generate_client_uuid(self.name)
          self.fs_name = fs_name
+        # settings below here won't be seen by the MDSDEV code!
          if config_only:
              self.config_only = 1
              return
          self.config_only = None
          mds = self.db.lookup(self.mds_uuid)
          self.mds_name = mds.getName()
          if config_only:
              self.config_only = 1
              return
          self.config_only = None
          mds = self.db.lookup(self.mds_uuid)
          self.mds_name = mds.getName()
+        self.devlist = self.db.get_lov_tgts('lov_tgt')
          for (obd_uuid, index, gen, active) in self.devlist:
              if obd_uuid == '':
                  continue
          for (obd_uuid, index, gen, active) in self.devlist:
              if obd_uuid == '':
                  continue
-           self.obdlist.append(obd_uuid)
-           obd = self.db.lookup(obd_uuid)
+            obd = self.db.lookup(obd_uuid)
              osc = get_osc(obd, self.uuid, fs_name)
              if osc:
              osc = get_osc(obd, self.uuid, fs_name)
              if osc:
-                self.osclist.append((osc, index, gen, active))
+                 self.osclist.append((osc, index, gen, active))
              else:
                  panic('osc not found:', obd_uuid)
              else:
                  panic('osc not found:', obd_uuid)
-    def get_uuid(self):
-        return self.uuid
-    def get_name(self):
-        return self.name
+        if self.osclist == []:
+            debug("get_lov_tgts failed, using get_refs");
+            index = 0
+            self.devlist = self.db.get_refs('obd')
+            for obd_uuid in self.devlist:
+                obd = self.db.lookup(obd_uuid)
+                osc = get_osc(obd, self.uuid, fs_name)
+                if osc:
+                    self.osclist.append((osc, index, 1, 1))
+                else:
+                    panic('osc not found:', obd_uuid)
+                index = index + 1
+        if self.osclist == []:
+            panic('No OSCs configured for LOV')
+        debug('dbg LOV __init__:', self.osclist, self.devlist, self.stripe_cnt)
+
      def prepare(self):
      def prepare(self):
-        if not config.record and is_prepared(self.name):
+        debug('dbg LOV prepare')
+        if is_prepared(self.name):
              return
              return
+        debug('dbg LOV prepare:', self.osclist, self.devlist)
          self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
                    self.stripe_off, self.pattern, self.devlist,
                    self.mds_name)
          self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
                    self.stripe_off, self.pattern, self.devlist,
                    self.mds_name)
-        lctl.lov_setup(self.name, self.uuid, self.desc_uuid,  self.stripe_cnt,
-                       self.stripe_sz, self.stripe_off, self.pattern,
-                      string.join(self.obdlist))
+        lctl.lov_setup(self.name, self.uuid,
+                       self.desc_uuid, self.mds_name, self.stripe_cnt,
+                       self.stripe_sz, self.stripe_off, self.pattern)
+        if self.osclist == []:
+            panic('No OSCs configured for LOV?')
          for (osc, index, gen, active) in self.osclist:
              target_uuid = osc.target_uuid
              try:
          for (osc, index, gen, active) in self.osclist:
              target_uuid = osc.target_uuid
              try:
@@ -1614,240 +1244,122 @@ class LOV(Module):
              lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
  
      def cleanup(self):
              lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
  
      def cleanup(self):
-        for (osc, index, gen, active) in self.osclist:
-            target_uuid = osc.target_uuid
-            osc.cleanup()
          if is_prepared(self.name):
              Module.cleanup(self)
          if is_prepared(self.name):
              Module.cleanup(self)
+        for (osc, index, gen, active) in self.osclist:
+            osc.cleanup()
          if self.config_only:
              panic("Can't clean up config_only LOV ", self.name)
  
          if self.config_only:
              panic("Can't clean up config_only LOV ", self.name)
  
-    def add_module(self, manager):
+    def load_module(self):
          if self.config_only:
              panic("Can't load modules for config_only LOV ", self.name)
          for (osc, index, gen, active) in self.osclist:
          if self.config_only:
              panic("Can't load modules for config_only LOV ", self.name)
          for (osc, index, gen, active) in self.osclist:
-            osc.add_module(manager)
+            osc.load_module()
              break
              break
-        manager.add_lustre_module('lov', 'lov')
-
-    def correct_level(self, level, op=None):
-        return level
+        Module.load_module(self)
  
  
-class LMV(Module):
-    def __init__(self, db, uuid, fs_name, name_override = None):
-        Module.__init__(self, 'LMV', db)
-        if name_override != None:
-            self.name = "lmv_%s" % name_override
-           
-        self.devlist = self.db.get_lmv_tgts('lmv_tgt')
-       if self.devlist == None:
-           self.devlist = self.db.get_refs('mds')
-           
-        self.mdclist = []
-        self.desc_uuid = self.uuid
-        self.uuid = uuid
-        self.fs_name = fs_name
-        for mds_uuid in self.devlist:
-            mds = self.db.lookup(mds_uuid)
-           if not mds:
-               panic("MDS not found!")
-           mdc = MDC(mds, self.uuid, fs_name)
-           if mdc:
-                self.mdclist.append(mdc)
-            else:
-                panic('mdc not found:', mds_uuid)
-
-    def prepare(self):
-        if is_prepared(self.name):
-            return
-           
-       self.info();
-        for mdc in self.mdclist:
-            try:
-                # Only ignore connect failures with --force, which
-                # isn't implemented here yet.
-                mdc.prepare(ignore_connect_failure=0)
-            except CommandError, e:
-                print "Error preparing LMV %s\n" % mdc.uuid
-                raise e
-       
-        lctl.lmv_setup(self.name, self.uuid, self.desc_uuid,
-                       string.join(self.devlist))
-
-    def cleanup(self):
-        for mdc in self.mdclist:
-            mdc.cleanup()
-        if is_prepared(self.name):
-            Module.cleanup(self)
-
-    def add_module(self, manager):
-        for mdc in self.mdclist:
-            mdc.add_module(manager)
+    def cleanup_module(self):
+        if self.config_only:
+            panic("Can't cleanup modules for config_only LOV ", self.name)
+        Module.cleanup_module(self)
+        for (osc, index, gen, active) in self.osclist:
+            if active:
+                osc.cleanup_module()
              break
              break
-        manager.add_lustre_module('lmv', 'lmv')
-
-    def correct_level(self, level, op=None):
-        return level
  
  class MDSDEV(Module):
      def __init__(self,db):
          Module.__init__(self, 'MDSDEV', db)
          self.devpath = self.db.get_val('devpath','')
  
  class MDSDEV(Module):
      def __init__(self,db):
          Module.__init__(self, 'MDSDEV', db)
          self.devpath = self.db.get_val('devpath','')
-        self.backdevpath = self.db.get_val('backdevpath','')
          self.size = self.db.get_val_int('devsize', 0)
          self.journal_size = self.db.get_val_int('journalsize', 0)
          self.size = self.db.get_val_int('devsize', 0)
          self.journal_size = self.db.get_val_int('journalsize', 0)
+
          self.fstype = self.db.get_val('fstype', '')
          self.fstype = self.db.get_val('fstype', '')
-        self.backfstype = self.db.get_val('backfstype', '')
+        if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
+            self.fstype = 'ext3'
+        elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
+            self.fstype = 'ldiskfs'
+
          self.nspath = self.db.get_val('nspath', '')
          self.nspath = self.db.get_val('nspath', '')
-        self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+        self.mkfsoptions = '-i 4096 ' + self.db.get_val('mkfsoptions', '')
          self.mountfsoptions = self.db.get_val('mountfsoptions', '')
          self.mountfsoptions = self.db.get_val('mountfsoptions', '')
-        self.obdtype = self.db.get_val('obdtype', '')
-        self.root_squash = self.db.get_val('root_squash', '')
-        self.no_root_squash = self.db.get_val('no_root_squash', '')
-       # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
+        if config.quota:
+            self.quota = config.quota
+        else:
+            self.quota = self.db.get_val('quota', '')
+        # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
          target_uuid = self.db.get_first_ref('target')
          target_uuid = self.db.get_first_ref('target')
-        self.mds = self.db.lookup(target_uuid)
-        self.name = self.mds.getName()
-        self.client_uuids = self.mds.get_refs('client')
-        
-       self.lmv = None
-        self.master = None
-       
-        lmv_uuid = self.db.get_first_ref('lmv')
-       if lmv_uuid != None:
-           self.lmv = self.db.lookup(lmv_uuid)
-           if self.lmv != None:
-                self.client_uuids = self.lmv.get_refs('client')
-
+        mds = self.db.lookup(target_uuid)
+        self.name = mds.getName()
+        self.filesystem_uuids = mds.get_refs('filesystem')
          # FIXME: if fstype not set, then determine based on kernel version
          self.format = self.db.get_val('autoformat', "no")
          # FIXME: if fstype not set, then determine based on kernel version
          self.format = self.db.get_val('autoformat', "no")
-        if self.mds.get_val('failover', 0):
+        if mds.get_val('failover', '1') != '0':
              self.failover_mds = 'f'
          else:
              self.failover_mds = 'n'
              self.failover_mds = 'f'
          else:
              self.failover_mds = 'n'
-        active_uuid = get_active_target(self.mds)
+        active_uuid = get_active_target(mds)
          if not active_uuid:
              panic("No target device found:", target_uuid)
          if active_uuid == self.uuid:
              self.active = 1
          else:
              self.active = 0
          if not active_uuid:
              panic("No target device found:", target_uuid)
          if active_uuid == self.uuid:
              self.active = 1
          else:
              self.active = 0
-        if self.active and config.group and config.group != self.mds.get_val('group'):
+        if self.active and config.group and config.group != mds.get_val('group', mds.get_val('name')):
              self.active = 0
  
              self.active = 0
  
-        # default inode inode for case when neither LOV either 
-        # LMV is accessible.
-        self.inode_size = 256
-        
-        inode_size = self.db.get_val_int('inodesize', 0)
-        if not inode_size == 0:
-            self.inode_size = inode_size
-        else:
+        self.inode_size = self.db.get_val_int('inodesize', 0)
+        debug('original inode_size ', self.inode_size)
+        if self.inode_size == 0:
              # find the LOV for this MDS
              # find the LOV for this MDS
-            lovconfig_uuid = self.mds.get_first_ref('lovconfig')
-            if lovconfig_uuid or self.lmv != None:
-                if self.lmv != None:
-                    lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
-                    lovconfig = self.lmv.lookup(lovconfig_uuid)
-                    lov_uuid = lovconfig.get_first_ref('lov')
-                    if lov_uuid == None:
-                        panic(self.mds.getName() + ": No LOV found for lovconfig ", 
-                              lovconfig.name)
-               else:
-                    lovconfig = self.mds.lookup(lovconfig_uuid)
-                    lov_uuid = lovconfig.get_first_ref('lov')
-                    if lov_uuid == None:
-                       panic(self.mds.getName() + ": No LOV found for lovconfig ", 
-                             lovconfig.name)
-
-                   if self.lmv != None:
-                       lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
-                       lovconfig = self.lmv.lookup(lovconfig_uuid)
-                       lov_uuid = lovconfig.get_first_ref('lov')
-
-                lov = LOV(self.db.lookup(lov_uuid), lov_uuid, self.name, 
-                          config_only = 1)
-
-                # default stripe count controls default inode_size
+            lovconfig_uuid = mds.get_first_ref('lovconfig')
+            if not lovconfig_uuid:
+                panic("No LOV config found for MDS ", mds.name)
+            lovconfig = mds.lookup(lovconfig_uuid)
+            lov_uuid = lovconfig.get_first_ref('lov')
+            if not lov_uuid:
+                panic("No LOV found for lovconfig ", lovconfig.name)
+            lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1)
+
+            # default stripe count controls default inode_size
+            if (lov.stripe_cnt > 0):
                  stripe_count = lov.stripe_cnt
                  stripe_count = lov.stripe_cnt
-                if stripe_count > 77:
-                    self.inode_size = 4096
-                elif stripe_count > 35:
-                    self.inode_size = 2048
-                elif stripe_count > 13:
-                    self.inode_size = 1024
-                elif stripe_count > 3:
-                    self.inode_size = 512
-                else:
-                    self.inode_size = 256
+            else:
+                stripe_count = 1
+            if stripe_count > 77:
+                self.inode_size = 512
+            elif stripe_count > 34:
+                self.inode_size = 2048
+            elif stripe_count > 13:
+                self.inode_size = 1024
+            #elif stripe_count < 3:
+            #    self.inode_size = 256
+            else:
+                self.inode_size = 512
+            debug('stripe_count ', stripe_count,' inode_size ',self.inode_size)
  
          self.target_dev_uuid = self.uuid
          self.uuid = target_uuid
  
  
          self.target_dev_uuid = self.uuid
          self.uuid = target_uuid
  
-       # setup LMV
-       if self.lmv != None:
-           client_uuid = self.name + "_lmv_UUID"
-           self.master = LMV(self.lmv, client_uuid, 
-                             self.name, self.name)
-
-    def add_module(self, manager):
+        # loading modules
+        if self.quota:
+            self.add_lustre_module('quota', 'lquota')
+        self.add_lustre_module('mdc', 'mdc')
+        self.add_lustre_module('osc', 'osc')
+        self.add_lustre_module('lov', 'lov')
+        self.add_lustre_module('mds', 'mds')
+        if self.fstype == 'ldiskfs':
+            self.add_lustre_module('ldiskfs', 'ldiskfs')
+        if self.fstype:
+            self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
+
+    def load_module(self):
          if self.active:
          if self.active:
-            manager.add_lustre_module('mdc', 'mdc')
-            manager.add_lustre_module('osc', 'osc')
-            manager.add_lustre_module('ost', 'ost')
-            manager.add_lustre_module('lov', 'lov')
-            manager.add_lustre_module('mds', 'mds')
-
-            if self.fstype == 'smfs' or self.fstype == 'ldiskfs':
-                manager.add_lustre_module(self.fstype, self.fstype)
-               
-            if self.fstype:
-                manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
-            
-            # if fstype is smfs, then we should also take care about backing 
-            # store fs.
-            if self.fstype == 'smfs':
-                manager.add_lustre_module(self.backfstype, self.backfstype)
-                manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.backfstype))
-
-           for option in string.split(self.mountfsoptions, ','):
-               if option == 'snap':
-                   if not self.fstype == 'smfs':
-                       panic("mountoptions has 'snap', but fstype is not smfs.")
-                   manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
-                   manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
-
-       # add LMV modules
-       if self.master != None:
-            self.master.add_module(manager)
-           
-    def get_mount_options(self, blkdev):
-        options = def_mount_options(self.fstype, 'mds')
-            
-        if config.mountfsoptions:
-            if options:
-                options = "%s,%s" %(options, config.mountfsoptions)
-            else:
-                options = config.mountfsoptions
-            if self.mountfsoptions:
-                options = "%s,%s" %(options, self.mountfsoptions)
-        else:
-            if self.mountfsoptions:
-                if options:
-                    options = "%s,%s" %(options, self.mountfsoptions)
-                else:
-                    options = self.mountfsoptions
-            
-        if self.fstype == 'smfs':
-            if options:
-                options = "%s,type=%s,dev=%s" %(options, 
-                          self.backfstype, blkdev)
-            else:
-                options = "type=%s,dev=%s" %(self.backfstype, blkdev)
-       return options
-       
+            Module.load_module(self)
+
      def prepare(self):
      def prepare(self):
-        if not config.record and is_prepared(self.name):
+        if is_prepared(self.name):
              return
          if not self.active:
              debug(self.uuid, "not active")
              return
          if not self.active:
              debug(self.uuid, "not active")
@@ -1855,136 +1367,72 @@ class MDSDEV(Module):
          if config.reformat:
              # run write_conf automatically, if --reformat used
              self.write_conf()
          if config.reformat:
              # run write_conf automatically, if --reformat used
              self.write_conf()
-        run_acceptors()
-        
-       # prepare LMV
-       if self.master != None:
-             self.master.prepare()
-            
+        self.info(self.devpath, self.fstype, self.size, self.format)
          # never reformat here
          blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
                             self.format, self.journal_size, self.inode_size,
          # never reformat here
          blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
                             self.format, self.journal_size, self.inode_size,
-                           self.mkfsoptions, self.backfstype, self.backdevpath)
-        
+                           self.mkfsoptions)
          if not is_prepared('MDT'):
              lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
          if not is_prepared('MDT'):
              lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
-        try: 
-            if self.fstype == 'smfs':
-                realdev = self.fstype
-            else:
-                realdev = blkdev
-               
-            if self.obdtype == None:
-                self.obdtype = 'dumb'
-               
-           if self.master == None:
-               master_name = 'dumb'
-           else:
-               master_name = self.master.name
-                
-            if self.client_uuids == None:
-               profile_name = 'dumb'
-           else:
-               profile_name = self.name
-            
-           mountfsoptions = self.get_mount_options(blkdev)
-
-           self.info("mds", realdev, mountfsoptions, self.fstype, self.size, 
-                     self.format, master_name, profile_name, self.obdtype)
-           
-           lctl.newdev("mds", self.name, self.uuid,
-                       setup = "%s %s %s %s %s %s" %(realdev, 
-                           self.fstype, profile_name, mountfsoptions,
-                            master_name, self.obdtype))
-
-            if development_mode():
-                procentry = "/proc/fs/lustre/mds/grp_hash_upcall"
-                upcall = os.path.abspath(os.path.dirname(sys.argv[0]) + "/l_getgroups")
-                if not (os.access(procentry, os.R_OK) and os.access(upcall, os.R_OK)):
-                    print "MDS Warning: failed to set group-hash upcall"
+        try:
+            mountfsoptions = def_mount_options(self.fstype, 'mds', blkdev)
+
+            if config.mountfsoptions:
+                if mountfsoptions:
+                    mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
                  else:
                  else:
-                    run("echo ", upcall, " > ", procentry)
+                    mountfsoptions = config.mountfsoptions
+                if self.mountfsoptions:
+                    mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+            else:
+                if self.mountfsoptions:
+                    if mountfsoptions:
+                        mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+                    else:
+                        mountfsoptions = self.mountfsoptions
+
+            print 'MDS mount options: ' + mountfsoptions
+
+            lctl.newdev("mds", self.name, self.uuid,
+                        setup ="%s %s %s %s %s" %(blkdev, self.fstype, self.name, 
+                                                  mountfsoptions, self.quota))
+            self.group_upcall = self.db.get_val('group_upcall','')
+            sys_set_group_upcall(self.name, self.group_upcall)
  
          except CommandError, e:
              if e.rc == 2:
  
          except CommandError, e:
              if e.rc == 2:
-                panic("MDS is missing the config log. Need to run " +
-                       "lconf --write_conf.")
+                panic("MDS failed to start.  Check the syslog for details." +
+                      " (May need to run lconf --write-conf)")
              else:
                  raise e
              else:
                  raise e
-        
-       if config.root_squash == None:
-            config.root_squash = self.root_squash
-        if config.no_root_squash == None:
-            config.no_root_squash = self.no_root_squash
-        if config.root_squash:
-            if config.no_root_squash:
-                nsnid = config.no_root_squash
-            else:
-                nsnid = "0"
-            lctl.root_squash(self.name, config.root_squash, nsnid)
  
      def write_conf(self):
  
      def write_conf(self):
-        if not self.client_uuids:
-            return 0
-            
-        do_cleanup = 0
-        if not is_prepared(self.name):
-            blkdev = block_dev(self.devpath, self.size, self.fstype,
-                               config.reformat, self.format, self.journal_size,
-                               self.inode_size, self.mkfsoptions,
-                               self.backfstype, self.backdevpath)
-
-            if self.fstype == 'smfs':
-                realdev = self.fstype
-            else:
-                realdev = blkdev
-            
-            # Even for writing logs we mount mds with supplied mount options
-            # because it will not mount smfs (if used) otherwise.
-           mountfsoptions = self.get_mount_options(blkdev)
-
-            if self.obdtype == None:
-                self.obdtype = 'dumb'
-                
-           self.info("mds", realdev, mountfsoptions, self.fstype, self.size, 
-                     self.format, "dumb", "dumb", self.obdtype)
-           
-            lctl.newdev("mds", self.name, self.uuid,
-                        setup ="%s %s %s %s %s %s" %(realdev, self.fstype, 
-                                                     'dumb', mountfsoptions,
-                                                     'dumb', self.obdtype))
-            do_cleanup = 1
-
-        # record logs for all MDS clients
-        for obd_uuid in self.client_uuids:
-            log("recording client:", obd_uuid)
-
+        if is_prepared(self.name):
+            return
+        self.info(self.devpath, self.fstype, self.format)
+        blkdev = block_dev(self.devpath, self.size, self.fstype,
+                           config.reformat, self.format, self.journal_size,
+                           self.inode_size, self.mkfsoptions)
+        lctl.newdev("mds", self.name, self.uuid,
+                    setup ="%s %s" %(blkdev, self.fstype))
+
+        # record logs for the MDS lov
+        for uuid in self.filesystem_uuids:
+            log("recording clients for filesystem:", uuid)
+            fs = self.db.lookup(uuid)
+            obd_uuid = fs.get_first_ref('obd')
              client_uuid = generate_client_uuid(self.name)
              client_uuid = generate_client_uuid(self.name)
-            client = VOSC(self.db.lookup(obd_uuid), client_uuid, 
-                          self.name, self.name)
+            client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
+                          self.name)
              config.record = 1
              lctl.clear_log(self.name, self.name)
              lctl.record(self.name, self.name)
              client.prepare()
              lctl.mount_option(self.name, client.get_name(), "")
              lctl.end_record()
              config.record = 1
              lctl.clear_log(self.name, self.name)
              lctl.record(self.name, self.name)
              client.prepare()
              lctl.mount_option(self.name, client.get_name(), "")
              lctl.end_record()
-            process_updates(self.db, self.name, self.name, client)
-
-            config.cleanup = 1
-            lctl.clear_log(self.name, self.name + '-clean')
-            lctl.record(self.name, self.name + '-clean')
-            client.cleanup()
-            lctl.del_mount_option(self.name)
-            lctl.end_record()
-            process_updates(self.db, self.name, self.name + '-clean', client)
-            config.cleanup = 0
              config.record = 0
  
          # record logs for each client
              config.record = 0
  
          # record logs for each client
-        if config.noexec:
-            noexec_opt = '-n'
-        else:
-            noexec_opt = ''
          if config.ldapurl:
              config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
          else:
          if config.ldapurl:
              config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
          else:
@@ -1994,54 +1442,75 @@ class MDSDEV(Module):
              client_name = node_db.getName()
              for prof_uuid in node_db.get_refs('profile'):
                  prof_db = node_db.lookup(prof_uuid)
              client_name = node_db.getName()
              for prof_uuid in node_db.get_refs('profile'):
                  prof_db = node_db.lookup(prof_uuid)
-                # refactor this into a funtion to test "clientness"
-                # of a node.
+                # refactor this into a funtion to test "clientness" of a node.
                  for ref_class, ref_uuid in prof_db.get_all_refs():
                      if ref_class in ('mountpoint','echoclient'):
                  for ref_class, ref_uuid in prof_db.get_all_refs():
                      if ref_class in ('mountpoint','echoclient'):
-                        debug("recording", client_name)
+                        thing = self.db.lookup(ref_uuid);
+                        fs_uuid = thing.get_first_ref('filesystem')
+                        if not fs_uuid in self.filesystem_uuids:
+                            continue;
+
+                        log("Recording log", client_name, "on", self.name)
                          old_noexec = config.noexec
                          config.noexec = 0
                          old_noexec = config.noexec
                          config.noexec = 0
-                        ret, out = run (sys.argv[0], noexec_opt,
-                                        " -v --record --nomod",
+                        noexec_opt = ('', '-n')
+                        ret, out = run (sys.argv[0],
+                                        noexec_opt[old_noexec == 1],
+                                        " -v --record --nomod --old_conf",
                                          "--record_log", client_name,
                                          "--record_device", self.name,
                                          "--node", client_name,
                                          config_options)
                                          "--record_log", client_name,
                                          "--record_device", self.name,
                                          "--node", client_name,
                                          config_options)
-                        if config.verbose:
-                            for s in out: log("record> ", string.strip(s))
-                        ret, out = run (sys.argv[0], noexec_opt,
-                                        "--cleanup -v --record --nomod",
-                                        "--record_log", client_name + "-clean",
-                                        "--record_device", self.name,
-                                        "--node", client_name,
-                                        config_options)
+                        if ret:
+                            lctl.clear_log(self.name, client_name)
+                            print out
+                            self.cleanup()
+                            panic("Record client log %s on %s failed" %(
+                                           client_name, self.name))
                          if config.verbose:
                              for s in out: log("record> ", string.strip(s))
                          config.noexec = old_noexec
                          if config.verbose:
                              for s in out: log("record> ", string.strip(s))
                          config.noexec = old_noexec
-        if do_cleanup:
-            try:
-                lctl.cleanup(self.name, self.uuid, 0, 0)
-            except CommandError, e:
-                log(self.module_name, "cleanup failed: ", self.name)
-                e.dump()
-                cleanup_error(e.rc)
-                Module.cleanup(self)
-
-            clean_dev(self.devpath, self.fstype, self.backfstype, 
-                     self.backdevpath)
+        try:
+            lctl.cleanup(self.name, self.uuid, config.force, config.failover)
+        except CommandError, e:
+            log(self.module_name, "cleanup failed: ", self.name)
+            e.dump()
+            cleanup_error(e.rc)
+            Module.cleanup(self)
+        clean_loop(self.devpath)
  
  
-    def msd_remaining(self):
+        #change the mtime of LLOG to match the XML creation time
+        if toplustreDB.get_mtime():
+            mtime = toplustreDB.get_mtime()
+            debug("changing mtime of LOGS to %s" %mtime)
+            ret, mktemp = runcmd("mktemp /tmp/lustre-cmd.XXXXXXXX")
+            if ret:
+                log(self.module_name, "create mtime LOGS cmdfile failed: ", self.name)
+            else:
+                mtimecmdfile = string.split(mktemp[0])[0]
+                fd = os.open(mtimecmdfile, os.O_RDWR | os.O_CREAT)
+                os.write(fd, "\n\n\n\n\n%s\n\n" %mtime)
+                os.close(fd)
+                cmd = "debugfs -w -R \"mi /LOGS\" <%s %s" %(mtimecmdfile, self.devpath)
+                ret, outs = runcmd(cmd)
+                os.remove(mtimecmdfile)
+                if ret:
+                    print "Can not change mtime of LOGS by debugfs."
+
+    def mds_remaining(self):
          out = lctl.device_list()
          for s in out:
              if string.split(s)[2] in ('mds',):
          out = lctl.device_list()
          for s in out:
              if string.split(s)[2] in ('mds',):
+                if string.split(s)[1] in ('ST',):
+                    return 0
                  return 1
  
      def safe_to_clean(self):
          return self.active
  
      def safe_to_clean_modules(self):
                  return 1
  
      def safe_to_clean(self):
          return self.active
  
      def safe_to_clean_modules(self):
-        return not self.msd_remaining()
-        
+        return not self.mds_remaining()
+
      def cleanup(self):
          if not self.active:
              debug(self.uuid, "not active")
      def cleanup(self):
          if not self.active:
              debug(self.uuid, "not active")
@@ -2056,10 +1525,7 @@ class MDSDEV(Module):
                  e.dump()
                  cleanup_error(e.rc)
                  Module.cleanup(self)
                  e.dump()
                  cleanup_error(e.rc)
                  Module.cleanup(self)
-           # cleanup LMV
-           if self.master != None:
-                self.master.cleanup()
-        if not self.msd_remaining() and is_prepared('MDT'):
+        if not self.mds_remaining() and is_prepared('MDT'):
              try:
                  lctl.cleanup("MDT", "MDT_UUID", config.force,
                               config.failover)
              try:
                  lctl.cleanup("MDT", "MDT_UUID", config.force,
                               config.failover)
@@ -2067,34 +1533,43 @@ class MDSDEV(Module):
                  print "cleanup failed: ", self.name
                  e.dump()
                  cleanup_error(e.rc)
                  print "cleanup failed: ", self.name
                  e.dump()
                  cleanup_error(e.rc)
-        
-        clean_dev(self.devpath, self.fstype, self.backfstype, 
-                 self.backdevpath)
-
-    def correct_level(self, level, op=None):
-       #if self.master != None:
-       #   level = level + 2
-        return level
+        clean_loop(self.devpath)
  
  class OSD(Module):
      def __init__(self, db):
          Module.__init__(self, 'OSD', db)
          self.osdtype = self.db.get_val('osdtype')
          self.devpath = self.db.get_val('devpath', '')
  
  class OSD(Module):
      def __init__(self, db):
          Module.__init__(self, 'OSD', db)
          self.osdtype = self.db.get_val('osdtype')
          self.devpath = self.db.get_val('devpath', '')
-        self.backdevpath = self.db.get_val('backdevpath', '')
          self.size = self.db.get_val_int('devsize', 0)
          self.journal_size = self.db.get_val_int('journalsize', 0)
          self.size = self.db.get_val_int('devsize', 0)
          self.journal_size = self.db.get_val_int('journalsize', 0)
+
+        # now as we store fids in EA on OST we need to make inode bigger
          self.inode_size = self.db.get_val_int('inodesize', 0)
          self.inode_size = self.db.get_val_int('inodesize', 0)
+        if self.inode_size == 0:
+                self.inode_size = 256
          self.mkfsoptions = self.db.get_val('mkfsoptions', '')
          self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+        # Allocate fewer inodes on large OST devices.  Most filesystems
+        # can be much more aggressive than this, but by default we can't.
+        if self.size > 1000000:
+                self.mkfsoptions = '-i 16384 ' + self.mkfsoptions
          self.mountfsoptions = self.db.get_val('mountfsoptions', '')
          self.mountfsoptions = self.db.get_val('mountfsoptions', '')
+        if config.quota:
+            self.quota = config.quota
+        else:
+            self.quota = self.db.get_val('quota', '')
+
          self.fstype = self.db.get_val('fstype', '')
          self.fstype = self.db.get_val('fstype', '')
-        self.backfstype = self.db.get_val('backfstype', '')
+        if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
+            self.fstype = 'ext3'
+        elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
+            self.fstype = 'ldiskfs'
+
          self.nspath = self.db.get_val('nspath', '')
          target_uuid = self.db.get_first_ref('target')
          ost = self.db.lookup(target_uuid)
          self.name = ost.getName()
          self.format = self.db.get_val('autoformat', 'yes')
          self.nspath = self.db.get_val('nspath', '')
          target_uuid = self.db.get_first_ref('target')
          ost = self.db.lookup(target_uuid)
          self.name = ost.getName()
          self.format = self.db.get_val('autoformat', 'yes')
-        if ost.get_val('failover', 0):
+        if ost.get_val('failover', '1') != '0':
              self.failover_ost = 'f'
          else:
              self.failover_ost = 'n'
              self.failover_ost = 'f'
          else:
              self.failover_ost = 'n'
@@ -2106,61 +1581,26 @@ class OSD(Module):
              self.active = 1
          else:
              self.active = 0
              self.active = 1
          else:
              self.active = 0
-        if self.active and config.group and config.group != ost.get_val('group'):
+        if self.active and config.group and config.group != ost.get_val('group', ost.get_val('name')):
              self.active = 0
  
          self.target_dev_uuid = self.uuid
          self.uuid = target_uuid
              self.active = 0
  
          self.target_dev_uuid = self.uuid
          self.uuid = target_uuid
-    
-    def add_module(self, manager):
+        # modules
+        if self.quota:
+            self.add_lustre_module('quota', 'lquota')
+        self.add_lustre_module('ost', 'ost')
+        # FIXME: should we default to ext3 here?
+        if self.fstype == 'ldiskfs':
+            self.add_lustre_module('ldiskfs', 'ldiskfs')
+        if self.fstype:
+            self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
+        self.add_lustre_module(self.osdtype, self.osdtype)
+
+    def load_module(self):
          if self.active:
          if self.active:
-            manager.add_lustre_module('ost', 'ost')
-            
-            if self.fstype == 'smfs' or self.fstype == 'ldiskfs':
-                manager.add_lustre_module(self.fstype, self.fstype)
-               
-            if self.fstype:
-                manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
-
-            if self.fstype == 'smfs':
-                manager.add_lustre_module(self.backfstype, self.backfstype)
-                manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.backfstype))
-
-           for option in self.mountfsoptions:
-               if option == 'snap':
-                   if not self.fstype == 'smfs':
-                       panic("mountoptions with snap, but fstype is not smfs\n")
-                   manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
-                   manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
-
-            manager.add_lustre_module(self.osdtype, self.osdtype)
-
-    def get_mount_options(self, blkdev):
-        options = def_mount_options(self.fstype, 'ost')
-            
-        if config.mountfsoptions:
-            if options:
-                options = "%s,%s" %(options, config.mountfsoptions)
-            else:
-                options = config.mountfsoptions
-            if self.mountfsoptions:
-                options = "%s,%s" %(options, self.mountfsoptions)
-        else:
-            if self.mountfsoptions:
-                if options:
-                    options = "%s,%s" %(options, self.mountfsoptions)
-                else:
-                    options = self.mountfsoptions
-            
-        if self.fstype == 'smfs':
-            if options:
-                options = "%s,type=%s,dev=%s" %(options, 
-                   self.backfstype, blkdev)
-            else:
-                options = "type=%s,dev=%s" %(self.backfstype, 
-                   blkdev)
-       return options
-       
+            Module.load_module(self)
+
      # need to check /proc/mounts and /etc/mtab before
      # formatting anything.
      # FIXME: check if device is already formatted.
      # need to check /proc/mounts and /etc/mtab before
      # formatting anything.
      # FIXME: check if device is already formatted.
@@ -2170,29 +1610,37 @@ class OSD(Module):
          if not self.active:
              debug(self.uuid, "not active")
              return
          if not self.active:
              debug(self.uuid, "not active")
              return
-        run_acceptors()
+        self.info(self.osdtype, self.devpath, self.size, self.fstype,
+                  self.format, self.journal_size, self.inode_size)
          if self.osdtype == 'obdecho':
              blkdev = ''
          else:
              blkdev = block_dev(self.devpath, self.size, self.fstype,
                                 config.reformat, self.format, self.journal_size,
          if self.osdtype == 'obdecho':
              blkdev = ''
          else:
              blkdev = block_dev(self.devpath, self.size, self.fstype,
                                 config.reformat, self.format, self.journal_size,
-                               self.inode_size, self.mkfsoptions, self.backfstype,
-                               self.backdevpath)
+                               self.inode_size, self.mkfsoptions)
  
  
-        if self.fstype == 'smfs':
-            realdev = self.fstype
+        mountfsoptions = def_mount_options(self.fstype, 'ost', blkdev)
+
+        if config.mountfsoptions:
+            if mountfsoptions:
+                mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+            else:
+                mountfsoptions = config.mountfsoptions
+            if self.mountfsoptions:
+                mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
          else:
          else:
-            realdev = blkdev
+            if self.mountfsoptions:
+                if mountfsoptions:
+                    mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+                else:
+                    mountfsoptions = self.mountfsoptions
+
+        print 'OST mount options: ' + mountfsoptions
  
  
-       mountfsoptions = self.get_mount_options(blkdev)
-       
-        self.info(self.osdtype, realdev, mountfsoptions, self.fstype, 
-                 self.size, self.format, self.journal_size, self.inode_size)
-                 
          lctl.newdev(self.osdtype, self.name, self.uuid,
          lctl.newdev(self.osdtype, self.name, self.uuid,
-                    setup ="%s %s %s %s" %(realdev, self.fstype,
-                                           self.failover_ost, 
-                                           mountfsoptions))
+                    setup ="%s %s %s %s %s" %(blkdev, self.fstype,
+                                           self.failover_ost, mountfsoptions,
+                                           self.quota))
          if not is_prepared('OSS'):
              lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
  
          if not is_prepared('OSS'):
              lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
  
@@ -2230,21 +1678,7 @@ class OSD(Module):
                  e.dump()
                  cleanup_error(e.rc)
          if not self.osdtype == 'obdecho':
                  e.dump()
                  cleanup_error(e.rc)
          if not self.osdtype == 'obdecho':
-           clean_dev(self.devpath, self.fstype, self.backfstype, 
-                     self.backdevpath)
-
-    def correct_level(self, level, op=None):
-        return level
-
-def mgmt_uuid_for_fs(mtpt_name):
-    if not mtpt_name:
-        return ''
-    mtpt_db = toplustreDB.lookup_name(mtpt_name)
-    fs_uuid = mtpt_db.get_first_ref('filesystem')
-    fs = toplustreDB.lookup(fs_uuid)
-    if not fs:
-        return ''
-    return fs.get_first_ref('mgmt')
+            clean_loop(self.devpath)
  
  # Generic client module, used by OSC and MDC
  class Client(Module):
  
  # Generic client module, used by OSC and MDC
  class Client(Module):
@@ -2252,15 +1686,14 @@ class Client(Module):
                   module_dir=None):
          self.target_name = tgtdb.getName()
          self.target_uuid = tgtdb.getUUID()
                   module_dir=None):
          self.target_name = tgtdb.getName()
          self.target_uuid = tgtdb.getUUID()
-        self.module_dir = module_dir
-       self.module = module
          self.db = tgtdb
          self.db = tgtdb
-        self.active = 1
+        self.backup_targets = []
  
          self.tgt_dev_uuid = get_active_target(tgtdb)
          if not self.tgt_dev_uuid:
  
          self.tgt_dev_uuid = get_active_target(tgtdb)
          if not self.tgt_dev_uuid:
-            panic("No target device found for target(1):", self.target_name)
+            panic("No target device found for target:", self.target_name)
  
  
+        self.kmod = kmod(config.lustre, config.portals)
          self._server = None
          self._connected = 0
  
          self._server = None
          self._connected = 0
  
@@ -2273,83 +1706,89 @@ class Client(Module):
              self.name = self_name
          self.uuid = uuid
          self.lookup_server(self.tgt_dev_uuid)
              self.name = self_name
          self.uuid = uuid
          self.lookup_server(self.tgt_dev_uuid)
-        mgmt_uuid = mgmt_uuid_for_fs(fs_name)
-        if mgmt_uuid:
-            self.mgmt_name = mgmtcli_name_for_uuid(mgmt_uuid)
-        else:
-            self.mgmt_name = ''
+        self.lookup_backup_targets()
          self.fs_name = fs_name
          self.fs_name = fs_name
-        if not self.module_dir:
-            self.module_dir = module
-
-    def add_module(self, manager):
-        manager.add_lustre_module(self.module_dir, self.module)
+        if not module_dir:
+            module_dir = module
+        self.add_lustre_module(module_dir, module)
  
      def lookup_server(self, srv_uuid):
          """ Lookup a server's network information """
          self._server_nets = get_ost_net(self.db, srv_uuid)
          if len(self._server_nets) == 0:
  
      def lookup_server(self, srv_uuid):
          """ Lookup a server's network information """
          self._server_nets = get_ost_net(self.db, srv_uuid)
          if len(self._server_nets) == 0:
-            panic ("Unable to find a server for:", srv_uuid)
-           
-    def get_name(self):
-        return self.name
+            panic("Unable to find a server for:", srv_uuid)
  
      def get_servers(self):
          return self._server_nets
  
  
      def get_servers(self):
          return self._server_nets
  
+    def lookup_backup_targets(self):
+        """ Lookup alternative network information """
+        prof_list = toplustreDB.get_refs('profile')
+        for prof_uuid in prof_list:
+            prof_db = toplustreDB.lookup(prof_uuid)
+            if not prof_db:
+                panic("profile:", prof_uuid, "not found.")
+            for ref_class, ref_uuid in prof_db.get_all_refs():
+                if ref_class in ('osd', 'mdsdev'):
+                    devdb = toplustreDB.lookup(ref_uuid)
+                    uuid = devdb.get_first_ref('target')
+                    if self.target_uuid == uuid and self.tgt_dev_uuid != ref_uuid:
+                        debug("add backup target", ref_uuid)
+                        self.backup_targets.append(ref_uuid)
+
      def prepare(self, ignore_connect_failure = 0):
          self.info(self.target_uuid)
      def prepare(self, ignore_connect_failure = 0):
          self.info(self.target_uuid)
-        if not config.record and is_prepared(self.name):
+        if is_prepared(self.name):
              self.cleanup()
          try:
              self.cleanup()
          try:
-            srv = choose_local_server(self.get_servers())
-            if srv:
+            srv_list = self.get_servers()
+            debug('dbg CLIENT __prepare__:', self.target_uuid, srv_list)
+            for srv in srv_list:
                  lctl.connect(srv)
                  lctl.connect(srv)
-            else:
-                routes = find_route(self.get_servers())
-                if len(routes) == 0:
-                    panic ("no route to",  self.target_uuid)
-                for (srv, r) in routes:
-                    lctl.add_route_host(r[0], srv.nid_uuid, r[1], r[3])
+            if len(srv_list) == 0:
+                panic("no servers for ", self.target_uuid)
          except CommandError, e:
              if not ignore_connect_failure:
                  raise e
          except CommandError, e:
              if not ignore_connect_failure:
                  raise e
-        if srv:
-            if self.permits_inactive() and (self.target_uuid in config.inactive or self.active == 0):
+
+        if srv_list[0]:
+            srv = srv_list[0]
+            if self.target_uuid in config.inactive and self.permits_inactive():
                  debug("%s inactive" % self.target_uuid)
                  inactive_p = "inactive"
              else:
                  debug("%s active" % self.target_uuid)
                  inactive_p = ""
              lctl.newdev(self.module, self.name, self.uuid,
                  debug("%s inactive" % self.target_uuid)
                  inactive_p = "inactive"
              else:
                  debug("%s active" % self.target_uuid)
                  inactive_p = ""
              lctl.newdev(self.module, self.name, self.uuid,
-                        setup ="%s %s %s %s" % (self.target_uuid, srv.nid_uuid,
-                                                inactive_p, self.mgmt_name))
+                        setup ="%s %s %s" % (self.target_uuid, srv.nid_uuid,
+                                                inactive_p))
+        else:
+            panic("Unable to create OSC for ", self.target_uuid)
+
+        for tgt_dev_uuid in self.backup_targets:
+            this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
+            if len(this_nets) == 0:
+                panic ("Unable to find a backup server for:", tgt_dev_uuid)
+            else:
+                for srv in this_nets:
+                    lctl.connect(srv)
+            if srv:
+                 lctl.add_conn(self.name, srv.nid_uuid);
+
  
      def cleanup(self):
          if is_prepared(self.name):
              Module.cleanup(self)
  
      def cleanup(self):
          if is_prepared(self.name):
              Module.cleanup(self)
-            try:
-                srv = choose_local_server(self.get_servers())
-                if srv:
-                    lctl.disconnect(srv)
+            srv_list = self.get_servers()
+            for srv in srv_list:
+                lctl.disconnect(srv)
+            for tgt_dev_uuid in self.backup_targets:
+                this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
+                if len(this_nets) == 0:
+                    panic ("Unable to find a backup server for:", tgt_dev_uuid)
                  else:
                  else:
-                    for (srv, r) in find_route(self.get_servers()):
-                        lctl.del_route_host(r[0], srv.nid_uuid, r[1], r[3])
-            except CommandError, e:
-                log(self.module_name, "cleanup failed: ", self.name)
-                e.dump()
-                cleanup_error(e.rc)
-
-    def correct_level(self, level, op=None):
-        return level
-
-    def deactivate(self):
-        try:
-            lctl.deactivate(self.name)
-        except CommandError, e:
-            log(self.module_name, "deactivate failed: ", self.name)
-            e.dump()
-            cleanup_error(e.rc)
+                    for srv in this_nets:
+                        lctl.disconnect(srv)
  
  class MDC(Client):
      def __init__(self, db, uuid, fs_name):
  
  class MDC(Client):
      def __init__(self, db, uuid, fs_name):
@@ -2365,242 +1804,62 @@ class OSC(Client):
      def permits_inactive(self):
          return 1
  
      def permits_inactive(self):
          return 1
  
-def mgmtcli_name_for_uuid(uuid):
-    return 'MGMTCLI_%s' % uuid
-
-class ManagementClient(Client):
-    def __init__(self, db, uuid):
-        Client.__init__(self, db, uuid, 'mgmt_cli', '',
-                        self_name = mgmtcli_name_for_uuid(db.getUUID()),
-                        module_dir = 'mgmt')
-
-class CMOBD(Module):
-    def __init__(self, db):
-       Module.__init__(self, 'CMOBD', db)
-       self.name = self.db.getName(); 
-       self.uuid = generate_client_uuid(self.name)
-       self.master_uuid = self.db.get_first_ref('masterobd')
-       self.cache_uuid = self.db.get_first_ref('cacheobd')
-
-       master_obd = self.db.lookup(self.master_uuid)
-       if not master_obd:
-           panic('master obd not found:', self.master_uuid)
-
-       cache_obd = self.db.lookup(self.cache_uuid)
-       if not cache_obd:
-           panic('cache obd not found:', self.cache_uuid)
-           
-       self.master = None
-       self.cache = None
-            
-        master_class = master_obd.get_class()
-        cache_class = cache_obd.get_class()
-
-       if master_class == 'ost' or master_class == 'lov':
-           client_uuid = "%s_lov_master_UUID" % (self.name)
-            self.master = LOV(master_obd, client_uuid, self.name);
-        elif master_class == 'mds':
-           self.master = get_mdc(db, self.name, self.master_uuid) 
-        elif master_class == 'lmv':
-           client_uuid = "%s_lmv_master_UUID" % (self.name)
-            self.master = LMV(master_obd, client_uuid, self.name);
-       else:
-           panic("unknown master obd class '%s'" %(master_class))
-           
-       if cache_class == 'ost' or cache_class == 'lov':
-           client_uuid = "%s_lov_cache_UUID" % (self.name)
-            self.cache = LOV(cache_obd, client_uuid, self.name);
-        elif cache_class == 'mds':
-           self.cache = get_mdc(db, self.name, self.cache_uuid)
-        elif cache_class == 'lmv':
-           client_uuid = "%s_lmv_cache_UUID" % (self.name)
-            self.cache = LMV(cache_obd, client_uuid, self.name);
-       else:
-           panic("unknown cache obd class '%s'" %(cache_class))
-
-    def prepare(self):
-        self.master.prepare()
-        if not config.record and is_prepared(self.name):
-            return
-        self.info(self.master_uuid, self.cache_uuid)
-        lctl.newdev("cmobd", self.name, self.uuid,
-                    setup ="%s %s" %(self.master.uuid,
-                                     self.cache.uuid))
-
-    def get_uuid(self):
-        return self.uuid
-       
-    def get_name(self):
-        return self.name
-       
-    def get_master_name(self):
-       return self.master.name
-           
-    def get_cache_name(self):
-        return self.cache.name
-
-    def cleanup(self):
-        if is_prepared(self.name):
-            Module.cleanup(self)
-       if self.master:
-           self.master.cleanup()
-
-    def add_module(self, manager):
-       manager.add_lustre_module('cmobd', 'cmobd')
-        self.master.add_module(manager)
-
-    def correct_level(self, level, op=None):
-        return level
-
  class COBD(Module):
  class COBD(Module):
-    def __init__(self, db, uuid, name):
+    def __init__(self, db):
          Module.__init__(self, 'COBD', db)
          Module.__init__(self, 'COBD', db)
-        self.name = self.db.getName(); 
-        self.uuid = generate_client_uuid(self.name)
-        self.master_uuid = self.db.get_first_ref('masterobd')
+        self.real_uuid = self.db.get_first_ref('realobd')
          self.cache_uuid = self.db.get_first_ref('cacheobd')
          self.cache_uuid = self.db.get_first_ref('cacheobd')
+        self.add_lustre_module('cobd' , 'cobd')
  
  
-        master_obd = self.db.lookup(self.master_uuid)
-        if not master_obd:
-            panic('master obd not found:', self.master_uuid)
-
-        cache_obd = self.db.lookup(self.cache_uuid)
-        if not cache_obd:
-            panic('cache obd not found:', self.cache_uuid)
-           
-       self.master = None
-       self.cache = None
-
-        master_class = master_obd.get_class()
-        cache_class = cache_obd.get_class()
-
-       if master_class == 'ost' or master_class == 'lov':
-           client_uuid = "%s_lov_master_UUID" % (self.name)
-            self.master = LOV(master_obd, client_uuid, name);
-        elif master_class == 'mds':
-            self.master = get_mdc(db, name, self.master_uuid) 
-        elif master_class == 'lmv':
-           client_uuid = "%s_lmv_master_UUID" % (self.name)
-            self.master = LMV(master_obd, client_uuid, self.name);
-       else:
-           panic("unknown master obd class '%s'" %(master_class))
-           
-       if cache_class == 'ost' or cache_class == 'lov':
-           client_uuid = "%s_lov_cache_UUID" % (self.name)
-            self.cache = LOV(cache_obd, client_uuid, name);
-        elif cache_class == 'mds':
-            self.cache = get_mdc(db, name, self.cache_uuid)
-        elif cache_class == 'lmv':
-           client_uuid = "%s_lmv_cache_UUID" % (self.name)
-            self.cache = LMV(cache_obd, client_uuid, self.name);
-       else:
-           panic("unknown cache obd class '%s'" %(cache_class))
-           
-    def get_uuid(self):
-        return self.uuid
-
-    def get_name(self):
-        return self.name
-
-    def get_master_name(self):
-        return self.master.name
-
-    def get_cache_name(self):
-        return self.cache.name
-
+    # need to check /proc/mounts and /etc/mtab before
+    # formatting anything.
+    # FIXME: check if device is already formatted.
      def prepare(self):
      def prepare(self):
-        self.master.prepare()
-        self.cache.prepare()
-        if not config.record and is_prepared(self.name):
+        if is_prepared(self.name):
              return
              return
-        self.info(self.master_uuid, self.cache_uuid)
+        self.info(self.real_uuid, self.cache_uuid)
          lctl.newdev("cobd", self.name, self.uuid,
          lctl.newdev("cobd", self.name, self.uuid,
-                    setup ="%s %s" %(self.master.name,
-                                     self.cache.name))
+                    setup ="%s %s" %(self.real_uuid, self.cache_uuid))
  
  
-    def cleanup(self):
-        if is_prepared(self.name):
-            Module.cleanup(self)
-        self.master.cleanup()
-        self.cache.cleanup()
-
-    def add_module(self, manager):
-        manager.add_lustre_module('cobd', 'cobd')
-        self.master.add_module(manager)
  
  # virtual interface for  OSC and LOV
  class VOSC(Module):
  
  # virtual interface for  OSC and LOV
  class VOSC(Module):
-    def __init__(self, db, client_uuid, name, name_override = None):
+    def __init__(self, db, uuid, fs_name, name_override = None, quota = None):
          Module.__init__(self, 'VOSC', db)
          Module.__init__(self, 'VOSC', db)
+        if quota:
+            self.add_lustre_module('quota', 'lquota')
          if db.get_class() == 'lov':
          if db.get_class() == 'lov':
-            self.osc = LOV(db, client_uuid, name, name_override)
-            self.type = 'lov'
-        elif db.get_class() == 'cobd':
-            self.osc = COBD(db, client_uuid, name)
-            self.type = 'cobd'
+            self.osc = LOV(db, uuid, fs_name, name_override)
          else:
          else:
-            self.osc = OSC(db, client_uuid, name)
-            self.type = 'osc'
-           
+            self.osc = get_osc(db, uuid, fs_name)
      def get_uuid(self):
      def get_uuid(self):
-        return self.osc.get_uuid()
-
+        return self.osc.uuid
      def get_name(self):
      def get_name(self):
-        return self.osc.get_name()
-
+        return self.osc.name
      def prepare(self):
          self.osc.prepare()
      def prepare(self):
          self.osc.prepare()
-       
      def cleanup(self):
          self.osc.cleanup()
      def cleanup(self):
          self.osc.cleanup()
-       
-    def add_module(self, manager):
-        self.osc.add_module(manager)
-       
-    def correct_level(self, level, op=None):
-        return self.osc.correct_level(level, op)
-
-# virtual interface for MDC and LMV
-class VMDC(Module):
-    def __init__(self, db, client_uuid, name, name_override = None):
-        Module.__init__(self, 'VMDC', db)
-        if db.get_class() == 'lmv':
-            self.mdc = LMV(db, client_uuid, name, name_override)
-        elif db.get_class() == 'cobd':
-            self.mdc = COBD(db, client_uuid, name)
-        else:
-            self.mdc = MDC(db, client_uuid, name)
-           
-    def get_uuid(self):
-        return self.mdc.uuid
+    def load_module(self):
+        Module.load_module(self)
+        self.osc.load_module()
+    def cleanup_module(self):
+        self.osc.cleanup_module()
+        Module.cleanup_module(self)
  
  
-    def get_name(self):
-        return self.mdc.name
-
-    def prepare(self):
-        self.mdc.prepare()
-       
-    def cleanup(self):
-        self.mdc.cleanup()
-       
-    def add_module(self, manager):
-        self.mdc.add_module(manager)
-       
-    def correct_level(self, level, op=None):
-        return self.mdc.correct_level(level, op)
  
  class ECHO_CLIENT(Module):
      def __init__(self,db):
          Module.__init__(self, 'ECHO_CLIENT', db)
  
  class ECHO_CLIENT(Module):
      def __init__(self,db):
          Module.__init__(self, 'ECHO_CLIENT', db)
+        self.add_lustre_module('obdecho', 'obdecho')
          self.obd_uuid = self.db.get_first_ref('obd')
          obd = self.db.lookup(self.obd_uuid)
          self.uuid = generate_client_uuid(self.name)
          self.osc = VOSC(obd, self.uuid, self.name)
  
      def prepare(self):
          self.obd_uuid = self.db.get_first_ref('obd')
          obd = self.db.lookup(self.obd_uuid)
          self.uuid = generate_client_uuid(self.name)
          self.osc = VOSC(obd, self.uuid, self.name)
  
      def prepare(self):
-        if not config.record and is_prepared(self.name):
+        if is_prepared(self.name):
              return
              return
-        run_acceptors()
          self.osc.prepare() # XXX This is so cheating. -p
          self.info(self.obd_uuid)
  
          self.osc.prepare() # XXX This is so cheating. -p
          self.info(self.obd_uuid)
  
@@ -2612,12 +1871,14 @@ class ECHO_CLIENT(Module):
              Module.cleanup(self)
          self.osc.cleanup()
  
              Module.cleanup(self)
          self.osc.cleanup()
  
-    def add_module(self, manager):
-        self.osc.add_module(manager)
-        manager.add_lustre_module('obdecho', 'obdecho')
+    def load_module(self):
+        self.osc.load_module()
+        Module.load_module(self)
+
+    def cleanup_module(self):
+        Module.cleanup_module(self)
+        self.osc.cleanup_module()
  
  
-    def correct_level(self, level, op=None):
-        return level
  
  def generate_client_uuid(name):
          client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
  
  def generate_client_uuid(name):
          client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
@@ -2626,73 +1887,78 @@ def generate_client_uuid(name):
                                                 int(random.random() * 1048576))
          return client_uuid[:36]
  
                                                 int(random.random() * 1048576))
          return client_uuid[:36]
  
+
+def my_rstrip(s, chars):
+    """my_rstrip(s, chars) -> strips any instances of the characters
+    found in chars from the right side of string s"""
+    # XXX required because python versions pre 2.2.3 don't allow
+    #string.rstrip() to take alternate char lists
+    import string
+    ns=s
+    try:
+        ns = string.rstrip(s, '/')
+    except TypeError, e:
+        for i in range(len(s) - 1, 0, -1):
+            if s[i] in chars:
+                continue
+            else:
+                ns = s[0:i+1]
+                break
+    return ns
+
+
  class Mountpoint(Module):
      def __init__(self,db):
          Module.__init__(self, 'MTPT', db)
  class Mountpoint(Module):
      def __init__(self,db):
          Module.__init__(self, 'MTPT', db)
-        self.path = self.db.get_val('path')
-       self.clientoptions = self.db.get_val('clientoptions', '')
+        self.path = my_rstrip(self.db.get_val('path'), '/')
+        self.clientoptions = self.db.get_val('clientoptions', '')
          self.fs_uuid = self.db.get_first_ref('filesystem')
          fs = self.db.lookup(self.fs_uuid)
          self.fs_uuid = self.db.get_first_ref('filesystem')
          fs = self.db.lookup(self.fs_uuid)
-        self.mds_uuid = fs.get_first_ref('lmv')
-       if not self.mds_uuid:
-           self.mds_uuid = fs.get_first_ref('mds')
+        self.mds_uuid = fs.get_first_ref('mds')
+        mds_db = self.db.lookup(self.mds_uuid)
+        if config.quota:
+            quota = config.quota
+        else:
+            quota = mds_db.get_val('quota', config.quota)
          self.obd_uuid = fs.get_first_ref('obd')
          self.obd_uuid = fs.get_first_ref('obd')
-        self.mgmt_uuid = fs.get_first_ref('mgmt')
+        obd = self.db.lookup(self.obd_uuid)
          client_uuid = generate_client_uuid(self.name)
          client_uuid = generate_client_uuid(self.name)
+        self.vosc = VOSC(obd, client_uuid, self.name, quota=quota)
+        self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
  
  
-        ost = self.db.lookup(self.obd_uuid)
-        if not ost:
-            panic("no ost: ", self.obd_uuid)
-            
-        mds = self.db.lookup(self.mds_uuid)
-       if not mds:
-           panic("no mds: ", self.mds_uuid)
-       
-        self.vosc = VOSC(ost, client_uuid, self.name, self.name)
-       self.vmdc = VMDC(mds, client_uuid, self.name, self.name)
-        
-        if self.mgmt_uuid:
-            self.mgmtcli = ManagementClient(db.lookup(self.mgmt_uuid),
-                                            client_uuid)
-        else:
-            self.mgmtcli = None
+        self.add_lustre_module('mdc', 'mdc')
+        self.add_lustre_module('llite', 'llite')
  
      def prepare(self):
  
      def prepare(self):
-        if not config.record and fs_is_mounted(self.path):
+        if fs_is_mounted(self.path):
              log(self.path, "already mounted.")
              return
              log(self.path, "already mounted.")
              return
-        run_acceptors()
-        if self.mgmtcli:
-            self.mgmtcli.prepare()
          self.vosc.prepare()
          self.vosc.prepare()
-        self.vmdc.prepare()
-        vmdc_name = self.vmdc.get_name()
+        self.mdc.prepare()
+        mdc_name = self.mdc.name
  
          self.info(self.path, self.mds_uuid, self.obd_uuid)
          if config.record or config.lctl_dump:
  
          self.info(self.path, self.mds_uuid, self.obd_uuid)
          if config.record or config.lctl_dump:
-            lctl.mount_option(local_node_name, self.vosc.get_name(), vmdc_name)
+            lctl.mount_option(local_node_name, self.vosc.get_name(), mdc_name)
              return
  
          if config.clientoptions:
              if self.clientoptions:
              return
  
          if config.clientoptions:
              if self.clientoptions:
-                self.clientoptions = self.clientoptions + ',' + \
-                                    config.clientoptions
+                self.clientoptions = self.clientoptions + ',' + config.clientoptions
              else:
                  self.clientoptions = config.clientoptions
          if self.clientoptions:
              self.clientoptions = ',' + self.clientoptions
              # Linux kernel will deal with async and not pass it to ll_fill_super,
              # so replace it with Lustre async
              else:
                  self.clientoptions = config.clientoptions
          if self.clientoptions:
              self.clientoptions = ',' + self.clientoptions
              # Linux kernel will deal with async and not pass it to ll_fill_super,
              # so replace it with Lustre async
-            self.clientoptions = string.replace(self.clientoptions, "async", 
-                                               "lasync")
+            self.clientoptions = string.replace(self.clientoptions, "async", "lasync")
  
          cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
  
          cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
-              (self.vosc.get_name(), vmdc_name, self.clientoptions, 
-              config.config, self.path)
+              (self.vosc.get_name(), mdc_name, self.clientoptions, config.config, self.path)
          run("mkdir", self.path)
          ret, val = run(cmd)
          if ret:
          run("mkdir", self.path)
          ret, val = run(cmd)
          if ret:
-            self.vmdc.cleanup()            
+            self.mdc.cleanup()
              self.vosc.cleanup()
              panic("mount failed:", self.path, ":", string.join(val))
  
              self.vosc.cleanup()
              panic("mount failed:", self.path, ":", string.join(val))
  
@@ -2713,24 +1979,17 @@ class Mountpoint(Module):
              if fs_is_mounted(self.path):
                  panic("fs is still mounted:", self.path)
  
              if fs_is_mounted(self.path):
                  panic("fs is still mounted:", self.path)
  
-        self.vmdc.cleanup()
+        self.mdc.cleanup()
          self.vosc.cleanup()
          self.vosc.cleanup()
-        if self.mgmtcli:
-            self.mgmtcli.cleanup()
  
  
-    def add_module(self, manager):
-        manager.add_lustre_module('mdc', 'mdc')
-        
-        if self.mgmtcli:
-            self.mgmtcli.add_module(manager)
-        
-        self.vosc.add_module(manager)
-        self.vmdc.add_module(manager)
+    def load_module(self):
+        self.vosc.load_module()
+        Module.load_module(self)
  
  
-        manager.add_lustre_module('llite', 'llite')
+    def cleanup_module(self):
+        Module.cleanup_module(self)
+        self.vosc.cleanup_module()
  
  
-    def correct_level(self, level, op=None):
-        return level
  
  # ============================================================
  # misc query functions
  
  # ============================================================
  # misc query functions
@@ -2744,31 +2003,26 @@ def get_ost_net(self, osd_uuid):
      node = self.lookup(node_uuid)
      if not node:
          panic("unable to find node for osd_uuid:", osd_uuid,
      node = self.lookup(node_uuid)
      if not node:
          panic("unable to find node for osd_uuid:", osd_uuid,
-              " node_ref:", node_uuid_)
+              " node_ref:", node_uuid)
      for net_uuid in node.get_networks():
          db = node.lookup(net_uuid)
      for net_uuid in node.get_networks():
          db = node.lookup(net_uuid)
-        srv_list.append(Network(db))
+        net = Network(db, node_uuid)
+        srv_list.append(net)
      return srv_list
  
  
      return srv_list
  
  
-# the order of iniitailization is based on level. 
+# the order of iniitailization is based on level.
  def getServiceLevel(self):
      type = self.get_class()
      ret=0;
      if type in ('network',):
          ret = 5
  def getServiceLevel(self):
      type = self.get_class()
      ret=0;
      if type in ('network',):
          ret = 5
-    elif type in ('routetbl',):
-        ret = 6
      elif type in ('ldlm',):
          ret = 20
      elif type in ('osd', 'cobd'):
          ret = 30
      elif type in ('mdsdev',):
          ret = 40
      elif type in ('ldlm',):
          ret = 20
      elif type in ('osd', 'cobd'):
          ret = 30
      elif type in ('mdsdev',):
          ret = 40
-    elif type in ('lmv',):
-        ret = 45
-    elif type in ('cmobd',):
-        ret = 50
      elif type in ('mountpoint', 'echoclient'):
          ret = 70
      else:
      elif type in ('mountpoint', 'echoclient'):
          ret = 70
      else:
@@ -2806,106 +2060,13 @@ def get_osc(ost_db, uuid, fs_name):
      osc = OSC(ost_db, uuid, fs_name)
      return osc
  
      osc = OSC(ost_db, uuid, fs_name)
      return osc
  
-def get_mdc(db, fs_name, mds_uuid):
+def get_mdc(db, uuid, fs_name, mds_uuid):
      mds_db = db.lookup(mds_uuid);
      if not mds_db:
      mds_db = db.lookup(mds_uuid);
      if not mds_db:
-        error("no mds:", mds_uuid)
-    mdc = MDC(mds_db, mds_uuid, fs_name)
+        panic("no mds:", mds_uuid)
+    mdc = MDC(mds_db, uuid, fs_name)
      return mdc
  
      return mdc
  
-############################################################
-# routing ("rooting")
-
-# list of (nettype, cluster_id, nid)
-local_clusters = []
-
-def find_local_clusters(node_db):
-    global local_clusters
-    for netuuid in node_db.get_networks():
-        net = node_db.lookup(netuuid)
-        srv = Network(net)
-        debug("add_local", netuuid)
-        local_clusters.append((srv.net_type, srv.cluster_id, srv.nid))
-        if srv.port > 0:
-            if acceptors.has_key(srv.port):
-                panic("duplicate port:", srv.port)
-            acceptors[srv.port] = AcceptorHandler(srv.port, srv.net_type)
-
-# This node is a gateway.
-is_router = 0
-def node_is_router():
-    return is_router
-
-# If there are any routers found in the config, then this will be true
-# and all nodes will load kptlrouter.
-needs_router = 0
-def node_needs_router():
-    return needs_router or is_router
-
-# list of (nettype, gw, tgt_cluster_id, lo, hi)
-# Currently, these local routes are only added to kptlrouter route
-# table if they are needed to connect to a specific server.  This
-# should be changed so all available routes are loaded, and the
-# ptlrouter can make all the decisions.
-local_routes = []
-
-def find_local_routes(lustre):
-    """ Scan the lustre config looking for routers .  Build list of
-    routes. """
-    global local_routes, needs_router
-    local_routes = []
-    list = lustre.lookup_class('node')
-    for router in list:
-        if router.get_val_int('router', 0):
-            needs_router = 1
-            for (local_type, local_cluster_id, local_nid) in local_clusters:
-                gw = None
-                for netuuid in router.get_networks():
-                    db = router.lookup(netuuid)
-                    if (local_type == db.get_val('nettype') and
-                       local_cluster_id == db.get_val('clusterid')):
-                        gw = db.get_val('nid')
-                        break
-                if gw:
-                    debug("find_local_routes: gw is", gw)
-                    for route in router.get_local_routes(local_type, gw):
-                        local_routes.append(route)
-    debug("find_local_routes:", local_routes)
-
-
-def choose_local_server(srv_list):
-    for srv in srv_list:
-        if local_cluster(srv.net_type, srv.cluster_id):
-            return srv
-
-def local_cluster(net_type, cluster_id):
-    for cluster in local_clusters:
-        if net_type == cluster[0] and cluster_id == cluster[1]:
-            return 1
-    return 0
-
-def local_interface(net_type, cluster_id, nid):
-    for cluster in local_clusters:
-        if (net_type == cluster[0] and cluster_id == cluster[1]
-            and nid == cluster[2]):
-            return 1
-    return 0
-
-def find_route(srv_list):
-    result = []
-    frm_type = local_clusters[0][0]
-    for srv in srv_list:
-        debug("find_route: srv:", srv.nid, "type: ", srv.net_type)
-        to_type = srv.net_type
-        to = srv.nid
-        cluster_id = srv.cluster_id
-        debug ('looking for route to', to_type, to)
-        for r in local_routes:
-            debug("find_route: ", r)
-            if  (r[3] <= to and to <= r[4]) and cluster_id == r[2]:
-                result.append((srv, r))
-    return result
-           
  def get_active_target(db):
      target_uuid = db.getUUID()
      target_name = db.getName()
  def get_active_target(db):
      target_uuid = db.getUUID()
      target_name = db.getName()
@@ -2921,7 +2082,7 @@ def get_server_by_nid_uuid(db,  nid_uuid):
          net = Network(n)
          if net.nid_uuid == nid_uuid:
              return net
          net = Network(n)
          if net.nid_uuid == nid_uuid:
              return net
-        
+
  
  ############################################################
  # lconf level logic
  
  ############################################################
  # lconf level logic
@@ -2936,29 +2097,23 @@ def newService(db):
          n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
      elif type == 'network':
          n = Network(db)
          n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
      elif type == 'network':
          n = Network(db)
-    elif type == 'routetbl':
-        n = RouteTable(db)
      elif type == 'osd':
          n = OSD(db)
      elif type == 'cobd':
      elif type == 'osd':
          n = OSD(db)
      elif type == 'cobd':
-        n = COBD(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
-    elif type == 'cmobd':
-        n = CMOBD(db)
+        n = COBD(db)
      elif type == 'mdsdev':
          n = MDSDEV(db)
      elif type == 'mountpoint':
          n = Mountpoint(db)
      elif type == 'echoclient':
          n = ECHO_CLIENT(db)
      elif type == 'mdsdev':
          n = MDSDEV(db)
      elif type == 'mountpoint':
          n = Mountpoint(db)
      elif type == 'echoclient':
          n = ECHO_CLIENT(db)
-    elif type == 'lmv':
-        n = LMV(db)
      else:
      else:
-        panic ("unknown service type:", type)
+        panic("unknown service type:", type)
      return n
  
  #
  # Prepare the system to run lustre using a particular profile
      return n
  
  #
  # Prepare the system to run lustre using a particular profile
-# in a the configuration. 
+# in a the configuration.
  #  * load & the modules
  #  * setup networking for the current node
  #  * make sure partitions are in place and prepared
  #  * load & the modules
  #  * setup networking for the current node
  #  * make sure partitions are in place and prepared
@@ -2972,240 +2127,169 @@ def for_each_profile(db, prof_list, operation):
          services = getServices(prof_db)
          operation(services)
  
          services = getServices(prof_db)
          operation(services)
  
-def magic_get_osc(db, rec, lov):
-    if lov:
-        lov_uuid = lov.get_uuid()
-        lov_name = lov.osc.fs_name
-    else:
-        lov_uuid = rec.getAttribute('lov_uuidref')
-        # FIXME: better way to find the mountpoint?
-        filesystems = db.root_node.getElementsByTagName('filesystem')
-        fsuuid = None
-        for fs in filesystems:
-            ref = fs.getElementsByTagName('obd_ref')
-            if ref[0].getAttribute('uuidref') == lov_uuid:
-                fsuuid = fs.getAttribute('uuid')
-                break
-
-        if not fsuuid:
-            panic("malformed xml: lov uuid '" + lov_uuid + "' referenced in 'add' record is not used by any filesystems.")
-
-        mtpts = db.root_node.getElementsByTagName('mountpoint')
-        lov_name = None
-        for fs in mtpts:
-            ref = fs.getElementsByTagName('filesystem_ref')
-            if ref[0].getAttribute('uuidref') == fsuuid:
-                lov_name = fs.getAttribute('name')
-                break
-
-        if not lov_name:
-            panic("malformed xml: 'add' record references lov uuid '" + lov_uuid + "', which references filesystem uuid '" + fsuuid + "', which does not reference a mountpoint.")
-
-    print "lov_uuid: " + lov_uuid + "; lov_name: " + lov_name
-
-    ost_uuid = rec.getAttribute('ost_uuidref')
-    obd = db.lookup(ost_uuid)
-
-    if not obd:
-        panic("malformed xml: 'add' record references ost uuid '" + ost_uuid + "' which cannot be found.")
-
-    osc = get_osc(obd, lov_uuid, lov_name)
-    if not osc:
-        panic('osc not found:', obd_uuid)
-    return osc
-
-# write logs for update records.  sadly, logs of all types -- and updates in
-# particular -- are something of an afterthought.  lconf needs rewritten with
-# these as core concepts.  so this is a pretty big hack.
-def process_update_record(db, update, lov):
-    for rec in update.childNodes:
-        if rec.nodeType != rec.ELEMENT_NODE:
-            continue
-
-        log("found "+rec.nodeName+" record in update version " +
-            str(update.getAttribute('version')))
-
-        lov_uuid = rec.getAttribute('lov_uuidref')
-        ost_uuid = rec.getAttribute('ost_uuidref')
-        index = rec.getAttribute('index')
-        gen = rec.getAttribute('generation')
-
-        if not lov_uuid or not ost_uuid or not index or not gen:
-            panic("malformed xml: 'update' record requires lov_uuid, ost_uuid, index, and generation.")
-
-        if not lov:
-            tmplov = db.lookup(lov_uuid)
-            if not tmplov:
-                panic("malformed xml: 'delete' record contains lov UUID '" + lov_uuid + "', which cannot be located.")
-            lov_name = tmplov.getName()
-        else:
-            lov_name = lov.osc.name
-
-        # ------------------------------------------------------------- add
-        if rec.nodeName == 'add':
-            if config.cleanup:
-                lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
-                continue
-
-            osc = magic_get_osc(db, rec, lov)
-
-            try:
-                # Only ignore connect failures with --force, which
-                # isn't implemented here yet.
-                osc.prepare(ignore_connect_failure=0)
-            except CommandError, e:
-                print "Error preparing OSC %s\n" % osc.uuid
-                raise e
-
-            lctl.lov_add_obd(lov_name, lov_uuid, ost_uuid, index, gen)
-
-        # ------------------------------------------------------ deactivate
-        elif rec.nodeName == 'deactivate':
-            if config.cleanup:
-                continue
-
-            osc = magic_get_osc(db, rec, lov)
-
-            try:
-                osc.deactivate()
-            except CommandError, e:
-                print "Error deactivating OSC %s\n" % osc.uuid
-                raise e
-
-        # ---------------------------------------------------------- delete
-        elif rec.nodeName == 'delete':
-            if config.cleanup:
-                continue
-
-            osc = magic_get_osc(db, rec, lov)
-
-            try:
-                config.cleanup = 1
-                osc.cleanup()
-                config.cleanup = 0
-            except CommandError, e:
-                print "Error cleaning up OSC %s\n" % osc.uuid
-                raise e
-
-            lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
-
-def process_updates(db, log_device, log_name, lov = None):
-    updates = db.root_node.getElementsByTagName('update')
-    for u in updates:
-        if not u.childNodes:
-            log("ignoring empty update record (version " +
-                str(u.getAttribute('version')) + ")")
-            continue
-
-        version = u.getAttribute('version')
-        real_name = "%s-%s" % (log_name, version)
-        lctl.clear_log(log_device, real_name)
-        lctl.record(log_device, real_name)
-
-        process_update_record(db, u, lov)
-
-        lctl.end_record()
-
  def doWriteconf(services):
  def doWriteconf(services):
-    #if config.nosetup:
-    #    return
+    if config.nosetup:
+        return
+    have_mds = 0
      for s in services:
          if s[1].get_class() == 'mdsdev':
              n = newService(s[1])
              n.write_conf()
      for s in services:
          if s[1].get_class() == 'mdsdev':
              n = newService(s[1])
              n.write_conf()
+            have_mds = 1
+    if have_mds == 0:
+        panic("Cannot find mds device, please run --write_conf on the mds node.")
+
  
  def doSetup(services):
      if config.nosetup:
          return
  
  def doSetup(services):
      if config.nosetup:
          return
-    slist = []
      for s in services:
          n = newService(s[1])
      for s in services:
          n = newService(s[1])
-       n.level = s[0]
-       slist.append((n.level, n))
-    nlist = []
-    for n in slist:
-       nl = n[1].correct_level(n[0])
-       nlist.append((nl, n[1]))
-    nlist.sort()
-    for n in nlist:
-        n[1].prepare()
-
-def doLoadModules(services):
+        n.prepare()
+
+def doModules(services):
      if config.nomod:
          return
      if config.nomod:
          return
-    
-    # adding all needed modules from all services
      for s in services:
          n = newService(s[1])
      for s in services:
          n = newService(s[1])
-        n.add_module(mod_manager)
-    
-    # loading all registered modules
-    mod_manager.load_modules()
+        n.load_module()
+
+def doCleanup(services):
+    if config.nosetup:
+        return
+    services.reverse()
+    for s in services:
+        n = newService(s[1])
+        if n.safe_to_clean():
+            n.cleanup()
  
  def doUnloadModules(services):
      if config.nomod:
          return
  
  def doUnloadModules(services):
      if config.nomod:
          return
-        
-    # adding all needed modules from all services
+    services.reverse()
      for s in services:
          n = newService(s[1])
          if n.safe_to_clean_modules():
      for s in services:
          n = newService(s[1])
          if n.safe_to_clean_modules():
-            n.add_module(mod_manager)
-    
-    # unloading all registered modules
-    mod_manager.cleanup_modules()
+            n.cleanup_module()
  
  
-def doCleanup(services):
+def doMakeServiceScript(services):
      if config.nosetup:
          return
      if config.nosetup:
          return
-    slist = []
-
+    try:
+        os.makedirs(config.service_scripts)
+    except OSError, e:
+        if e[0] != errno.EEXIST:
+            panic("Couldn't create scripts dir " + config.service_scripts + ": " + e[1])
+    
      for s in services:
      for s in services:
-        n = newService(s[1])
-       n.level = s[0]
-       slist.append((n.level, n))
-    nlist = []
-    for n in slist:
-       nl = n[1].correct_level(n[0])
-       nlist.append((nl, n[1]))
-    nlist.sort()
-    nlist.reverse()
-
-    for n in nlist:
-        if n[1].safe_to_clean():
-            n[1].cleanup()
+        if s[1].get_class() != 'osd' and s[1].get_class() != 'mdsdev':
+            continue
+
+        target_uuid = s[1].get_first_ref('target')
+        target = toplustreDB.lookup(target_uuid)
+        target_symlink = config.service_scripts + "/" + target.getName()
+        if config.force:
+            try:
+                try:
+                    os.unlink(target_symlink)
+                    if config.verbose:
+                        print "Removed " + target_symlink
+                except OSError, e:
+                    if e[0] != errno.EISDIR:
+                        raise e
+                    os.rmdir(target_symlink)
+                    if config.verbose:
+                        print "Removed " + target_symlink
+            except OSError, e:
+                if e[0] != errno.ENOENT:
+                    panic("Error removing " + target_symlink + ": " + e[1])
+                    
+        try:
+            os.symlink("/etc/init.d/lustre", target_symlink)
+            if config.verbose:
+                print "Created service link " + target_symlink + " to /etc/init.d/lustre"
+
+        except OSError, e:
+            if e[0] == errno.EEXIST:
+                extra_error = " (use --force option to remove existing files)"
+            else:
+                extra_error = ""
+            panic("Error creating " + target_symlink + ": " + e[1] + extra_error)
+
+# Check mtime of config logs
+def doCheckMtime(lustreDB, hosts):
+    for h in hosts:
+        node_db = lustreDB.lookup_name(h, 'node')
+        if node_db:
+            break
+    if not node_db:
+        return
+
+    mdsdb = 0
+    prof_list = node_db.get_refs('profile')
+    for prof_uuid in prof_list:
+        prof_db = node_db.lookup(prof_uuid)
+        if prof_db:
+            services = getServices(prof_db)
+            for s in services:
+               if s[1].get_class() == 'mdsdev':
+                  mdsdb = s[1]
+                  break
+
+    if mdsdb and lustreDB.get_mtime():
+        debug("Checking XML modification time")
+        devpath = mdsdb.get_val('devpath','')
+        xmtime = string.atol(lustreDB.get_mtime())
+        cmd = "debugfs -c -R 'stat /LOGS' %s 2>&1 | grep mtime" %devpath
+        ret, kmtimes = runcmd(cmd)
+        if ret:
+            log("Can not get mtime info of MDS LOGS directory")
+        else:
+            kmtime = string.atoi(string.split(kmtimes[0])[1], 0)
+            if xmtime > kmtime:
+                debug('xmtime ', xmtime, '> kmtime', kmtime)
+                if config.old_conf:
+                    log("Warning: MDS startup logs are older than config %s."
+                        " Please run --write_conf on stopped MDS to update."
+                        %CONFIG_FILE)
+                else:
+                    panic("Error: MDS startup logs are older than config %s."
+                          " Please run --write_conf on stopped MDS to update."
+                          " Use '--old_conf' to start anyways." %CONFIG_FILE)
+    return
  
  #
  
  #
-# Load profile for 
+# Load profile for
  def doHost(lustreDB, hosts):
  def doHost(lustreDB, hosts):
-    global is_router, local_node_name
+    global local_node_name, tgt_select
      node_db = None
      for h in hosts:
          node_db = lustreDB.lookup_name(h, 'node')
          if node_db:
      node_db = None
      for h in hosts:
          node_db = lustreDB.lookup_name(h, 'node')
          if node_db:
+            if config.service:
+                tgt_select[config.service] = h
+                config.group = config.service
              break
      if not node_db:
          panic('No host entry found.')
  
      local_node_name = node_db.get_val('name', 0)
              break
      if not node_db:
          panic('No host entry found.')
  
      local_node_name = node_db.get_val('name', 0)
-    is_router = node_db.get_val_int('router', 0)
      lustre_upcall = node_db.get_val('lustreUpcall', '')
      portals_upcall = node_db.get_val('portalsUpcall', '')
      timeout = node_db.get_val_int('timeout', 0)
      ptldebug = node_db.get_val('ptldebug', '')
      subsystem = node_db.get_val('subsystem', '')
      lustre_upcall = node_db.get_val('lustreUpcall', '')
      portals_upcall = node_db.get_val('portalsUpcall', '')
      timeout = node_db.get_val_int('timeout', 0)
      ptldebug = node_db.get_val('ptldebug', '')
      subsystem = node_db.get_val('subsystem', '')
-    
-    find_local_clusters(node_db)
-    if not is_router:
-        find_local_routes(lustreDB)
  
      # Two step process: (1) load modules, (2) setup lustre
      # if not cleaning, load modules first.
      prof_list = node_db.get_refs('profile')
  
  
      # Two step process: (1) load modules, (2) setup lustre
      # if not cleaning, load modules first.
      prof_list = node_db.get_refs('profile')
  
-    if config.write_conf:
-        for_each_profile(node_db, prof_list, doLoadModules)
-        sys_make_devices()
+    if config.make_service_scripts:
+        for_each_profile(node_db, prof_list, doMakeServiceScript)
+        return
+    
+    elif config.write_conf:
+        for_each_profile(node_db, prof_list, doModules)
          for_each_profile(node_db, prof_list, doWriteconf)
          for_each_profile(node_db, prof_list, doUnloadModules)
          lustreDB.close()
          for_each_profile(node_db, prof_list, doWriteconf)
          for_each_profile(node_db, prof_list, doUnloadModules)
          lustreDB.close()
@@ -3217,15 +2301,14 @@ def doHost(lustreDB, hosts):
          doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
                     config.conn_uuid)
      elif config.cleanup:
          doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
                     config.conn_uuid)
      elif config.cleanup:
-        if config.force:
-            # the command line can override this value
-            timeout = 5
+        if not mod_loaded('lnet'):
+            return
+
          # ugly hack, only need to run lctl commands for --dump
          if config.lctl_dump or config.record:
              for_each_profile(node_db, prof_list, doCleanup)
              return
  
          # ugly hack, only need to run lctl commands for --dump
          if config.lctl_dump or config.record:
              for_each_profile(node_db, prof_list, doCleanup)
              return
  
-        sys_set_timeout(timeout)
          sys_set_ptldebug(ptldebug)
          sys_set_subsystem(subsystem)
          sys_set_lustre_upcall(lustre_upcall)
          sys_set_ptldebug(ptldebug)
          sys_set_subsystem(subsystem)
          sys_set_lustre_upcall(lustre_upcall)
@@ -3243,15 +2326,17 @@ def doHost(lustreDB, hosts):
              for_each_profile(node_db, prof_list, doSetup)
              return
  
              for_each_profile(node_db, prof_list, doSetup)
              return
  
-        sys_make_devices()
-        sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
-        sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
+        if PLATFORM == 'LINUX':
+            sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
+            sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
  
  
-        for_each_profile(node_db, prof_list, doLoadModules)
+        for_each_profile(node_db, prof_list, doModules)
  
  
-        sys_set_debug_path()
-        sys_set_ptldebug(ptldebug)
-        sys_set_subsystem(subsystem)
+        if PLATFORM == 'LINUX':
+            # XXX need to be fixed for Darwin
+            sys_set_debug_path()
+            sys_set_ptldebug(ptldebug)
+            sys_set_subsystem(subsystem)
          script = config.gdb_script
          run(lctl.lctl, ' modules >', script)
          if config.gdb:
          script = config.gdb_script
          run(lctl.lctl, ' modules >', script)
          if config.gdb:
@@ -3266,6 +2351,150 @@ def doHost(lustreDB, hosts):
          for_each_profile(node_db, prof_list, doSetup)
          lustreDB.close()
  
          for_each_profile(node_db, prof_list, doSetup)
          lustreDB.close()
  
+def add_clumanager_node(node_db, nodes, services):
+    new_services = []
+    node_name = node_db.getUUID()
+    nodes[node_name] = []
+    
+    for prof_uuid in node_db.get_refs('profile'):
+        prof_db = toplustreDB.lookup(prof_uuid)
+        for ref_class, ref_uuid in prof_db.get_all_refs():
+            if ref_class not in ('osd', 'mdsdev'):
+                continue
+            devdb = toplustreDB.lookup(ref_uuid)
+            tgt_uuid = devdb.get_first_ref('target')
+
+            nodes[node_name].append(ref_uuid)
+
+            if not services.has_key(tgt_uuid):
+                if config.verbose:
+                    print "New service: " + tgt_uuid + " (originally found on " + node_name + ")"
+                new_services.append(tgt_uuid)
+                services[tgt_uuid] = []
+            services[tgt_uuid].append(ref_uuid)
+
+    return new_services
+
+def add_clumanager_services(new_services, nodes, dev_list):
+    new_nodes = []
+    for devdb in dev_list:
+        tgt_uuid = devdb.get_first_ref('target')
+        if tgt_uuid in new_services:
+            node_uuid = devdb.get_first_ref('node')
+        
+            if not (nodes.has_key(node_uuid) or node_uuid in new_nodes):
+                if config.verbose:
+                    print "New node: " + node_uuid + " for service " + tgt_uuid
+                new_nodes.append(node_uuid)
+
+    return new_nodes
+
+def doClumanager(lustreDB, hosts):
+    nodes = {}
+    services = {}
+
+    dev_list = []
+    
+    for dev_uuid in toplustreDB.get_refs('osd') + toplustreDB.get_refs('mdsdev'):
+        dev_list.append(lustreDB.lookup(dev_uuid))
+
+    node_db = None
+    for h in hosts:
+        node_db = lustreDB.lookup_name(h, 'node')
+        if node_db:
+            our_host = h
+            new_services = add_clumanager_node(node_db, nodes, services)
+            break
+            
+    if not node_db:
+        panic('No host entry found.')
+
+    while 1:
+        if len(new_services) == 0:
+            break
+        
+        new_nodes = add_clumanager_services(new_services, nodes, dev_list)
+        if len(new_nodes) == 0:
+            break
+
+        if len(new_nodes) + len(nodes.keys()) > 8:
+            panic("CluManager only supports 8 nodes per failover \"cluster.\"")
+
+        new_services = []
+        for node_uuid in new_nodes:
+            node_db = lustreDB.lookup(node_uuid)
+            if not node_db:
+                panic("No node entry for " + node_uuid + " was found.")
+
+            new_services.append(add_clumanager_node(node_db, nodes, services))
+
+    nodenames = []
+    for node in nodes.keys():
+        nodedb = lustreDB.lookup(node)
+        nodenames.append(nodedb.getName())
+    nodenames.sort()
+
+    print """<?xml version="1.0"?>
+<cluconfig version="3.0">
+  <clumembd broadcast="no" interval="750000" loglevel="5" multicast="yes" multicast_ipaddress="225.0.0.11" thread="yes" tko_count="20"/>
+  <cluquorumd loglevel="5" pinginterval="2"/>
+  <clurmtabd loglevel="5" pollinterval="4"/>
+  <clusvcmgrd loglevel="5"/>
+  <clulockd loglevel="5"/>
+  <cluster config_viewnumber="1" name="%s"/>
+  <sharedstate driver="libsharedraw.so" rawprimary="%s" rawshadow="%s" type="raw"/>
+  <members> """ % (string.join(nodenames), config.rawprimary, config.rawsecondary)
+    
+
+    i = 0
+    for node in nodenames:
+        print "    <member id=\"%d\" name=\"%s\" watchdog=\"yes\"/>" % (i, node)
+        i = i + 1
+
+    print "  </members>\n  <failoverdomains>"
+
+    servicekeys = services.keys()
+    servicekeys.sort()
+
+    i = 0
+    for service in servicekeys:
+        svcdb = lustreDB.lookup(service)
+        print "    <failoverdomain id=\"%d\" name=\"%s\" ordered=\"yes\" restricted=\"yes\">" % (i, svcdb.getName())
+        i = i + 1
+
+        j = 0
+        active_uuid = get_active_target(svcdb)
+        for svc_uuid in [active_uuid] + services[service]:
+            if svc_uuid == active_uuid and j > 0:
+                continue
+            svcdb = lustreDB.lookup(svc_uuid)
+
+            svc_node_uuid = svcdb.get_first_ref('node')
+            svc_nodedb = lustreDB.lookup(svc_node_uuid)
+
+            print "      <failoverdomainnode id=\"%d\" name=\"%s\"/>" % (j, svc_nodedb.getName())
+            j = j + 1
+
+        print "    </failoverdomain>"
+
+    print "  </failoverdomains>\n  <services>"
+
+    i = 0
+    for service in servicekeys:
+        svcdb = lustreDB.lookup(service)
+        active_uuid = get_active_target(svcdb)
+        activedb = lustreDB.lookup(active_uuid)
+
+        svc_node_uuid = activedb.get_first_ref('node')
+        svc_nodedb = lustreDB.lookup(svc_node_uuid)
+
+        print "    <service checkinterval=\"30\" failoverdomain=\"%s\" id=\"%d\" name=\"%s\" userscript=\"%s/%s\">" \
+              % ( svcdb.getName(), i, svcdb.getName(), config.service_scripts, svcdb.getName())
+        print "      <service_ipaddresses/>\n    </service>"
+        i = i + 1
+
+    print "  </services>\n</cluconfig>"
+
  def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
      tgt = lustreDB.lookup(tgt_uuid)
      if not tgt:
  def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
      tgt = lustreDB.lookup(tgt_uuid)
      if not tgt:
@@ -3274,34 +2503,27 @@ def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
      if not new_uuid:
          raise Lustre.LconfError("doRecovery: no active target found for: " +
                                  tgt_uuid)
      if not new_uuid:
          raise Lustre.LconfError("doRecovery: no active target found for: " +
                                  tgt_uuid)
-    net = choose_local_server(get_ost_net(lustreDB, new_uuid))
-    if not net:
+    srv_list = find_local_servers(get_ost_net(lustreDB, new_uuid))
+    if not srv_list[0]:
          raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
  
          raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
  
-    log("Reconnecting", tgt_uuid, " to ",  net.nid_uuid);
-    try:
-        oldnet = get_server_by_nid_uuid(lustreDB, nid_uuid)
-        lustreDB.close()
-        if oldnet:
-            lctl.disconnect(oldnet)
-    except CommandError, e:
-        log("recover: disconnect", nid_uuid, "failed: ")
-        e.dump()
+    oldsrv = get_server_by_nid_uuid(lustreDB, nid_uuid)
+    lustreDB.close()
  
  
-    try:
-        lctl.connect(net)
-    except CommandError, e:
-        log("recover: connect failed")
-        e.dump()
+    for srv in srv_list:
+        if oldsrv.net_type != srv.net_type:
+            continue
  
  
-    lctl.recover(client_uuid, net.nid_uuid)
+        log("Reconnecting", tgt_uuid, "to", srv.nid_uuid)
+
+        lctl.recover(client_uuid, srv.nid_uuid)
  
  
  def setupModulePath(cmd, portals_dir = PORTALS_DIR):
      base = os.path.dirname(cmd)
      if development_mode():
          if not config.lustre:
  
  
  def setupModulePath(cmd, portals_dir = PORTALS_DIR):
      base = os.path.dirname(cmd)
      if development_mode():
          if not config.lustre:
-            debug('using objdir module paths')            
+            debug('using objdir module paths')
              config.lustre = (os.path.join(base, ".."))
          # normalize the portals dir, using command line arg if set
          if config.portals:
              config.lustre = (os.path.join(base, ".."))
          # normalize the portals dir, using command line arg if set
          if config.portals:
@@ -3311,7 +2533,7 @@ def setupModulePath(cmd, portals_dir = PORTALS_DIR):
          debug('config.portals', config.portals)
      elif config.lustre and config.portals:
          # production mode
          debug('config.portals', config.portals)
      elif config.lustre and config.portals:
          # production mode
-        # if --lustre and --portals, normalize portals 
+        # if --lustre and --portals, normalize portals
          # can ignore POTRALS_DIR here, since it is probly useless here
          config.portals = os.path.join(config.lustre, config.portals)
          debug('config.portals B', config.portals)
          # can ignore POTRALS_DIR here, since it is probly useless here
          config.portals = os.path.join(config.lustre, config.portals)
          debug('config.portals B', config.portals)
@@ -3329,25 +2551,51 @@ def sysctl(path, val):
  
  
  def sys_set_debug_path():
  
  
  def sys_set_debug_path():
-    sysctl('portals/debug_path', config.debug_path)
+    sysctl('lnet/debug_path', config.debug_path)
+
+def validate_upcall(upcall):
+    import os
+    if upcall in ('DEFAULT','NONE'):
+        pass
+    elif os.path.exists(upcall):
+        if not os.access(upcall, os.X_OK):
+            print "WARNING upcall script not executable: %s" % upcall
+    else:
+        print "WARNING invalid upcall script specified: %s" % upcall
  
  def sys_set_lustre_upcall(upcall):
  
  def sys_set_lustre_upcall(upcall):
-    # the command overrides the value in the node config
+    # the command line overrides the value in the node config
      if config.lustre_upcall:
          upcall = config.lustre_upcall
      elif config.upcall:
          upcall = config.upcall
      if upcall:
      if config.lustre_upcall:
          upcall = config.lustre_upcall
      elif config.upcall:
          upcall = config.upcall
      if upcall:
+        validate_upcall(upcall)
          lctl.set_lustre_upcall(upcall)
  
  def sys_set_portals_upcall(upcall):
          lctl.set_lustre_upcall(upcall)
  
  def sys_set_portals_upcall(upcall):
-    # the command overrides the value in the node config
+    # the command line overrides the value in the node config
      if config.portals_upcall:
          upcall = config.portals_upcall
      elif config.upcall:
          upcall = config.upcall
      if upcall:
      if config.portals_upcall:
          upcall = config.portals_upcall
      elif config.upcall:
          upcall = config.upcall
      if upcall:
-        sysctl('portals/upcall', upcall)
+        validate_upcall(upcall)
+        sysctl('lnet/upcall', upcall)
+
+def sys_set_group_upcall(mds, upcall):
+    if config.noexec:
+        return
+    # the command line overrides the value in the MDS config
+    if config.group_upcall:
+        upcall = config.group_upcall
+    if upcall:
+        validate_upcall(upcall)
+        debug("setting MDS", mds, "upcall to:", upcall)
+        path = "/proc/fs/lustre/mds/" + mds + "/group_upcall"
+        fp = open(path, 'w')
+        fp.write(upcall)
+        fp.close()
  
  def sys_set_timeout(timeout):
      # the command overrides the value in the node config
  
  def sys_set_timeout(timeout):
      # the command overrides the value in the node config
@@ -3357,23 +2605,6 @@ def sys_set_timeout(timeout):
          lctl.set_timeout(timeout)
  
  def sys_tweak_socknal ():
          lctl.set_timeout(timeout)
  
  def sys_tweak_socknal ():
-    # reserve at least 8MB, or we run out of RAM in skb_alloc under read
-    if sys_get_branch() == '2.6':
-        fp = open('/proc/meminfo')
-        lines = fp.readlines()
-        fp.close()
-        memtotal = 131072
-        for l in lines:
-            a = string.split(l)
-            if a[0] == 'MemTotal:':
-                memtotal = a[1]
-               debug("memtotal" + memtotal)
-       if int(memtotal) < 262144:
-           minfree = int(memtotal) / 16
-       else:
-           minfree = 32768
-       debug("+ minfree ", minfree)
-        sysctl("vm/min_free_kbytes", minfree)
      if config.single_socket:
          sysctl("socknal/typed", 0)
  
      if config.single_socket:
          sysctl("socknal/typed", 0)
  
@@ -3392,7 +2623,7 @@ def sys_set_ptldebug(ptldebug):
          try:
              val = eval(ptldebug, ptldebug_names)
              val = "0x%x" % (val)
          try:
              val = eval(ptldebug, ptldebug_names)
              val = "0x%x" % (val)
-            sysctl('portals/debug', val)
+            sysctl('lnet/debug', val)
          except NameError, e:
              panic(str(e))
  
          except NameError, e:
              panic(str(e))
  
@@ -3403,7 +2634,7 @@ def sys_set_subsystem(subsystem):
          try:
              val = eval(subsystem, subsystem_names)
              val = "0x%x" % (val)
          try:
              val = eval(subsystem, subsystem_names)
              val = "0x%x" % (val)
-            sysctl('portals/subsystem_debug', val)
+            sysctl('lnet/subsystem_debug', val)
          except NameError, e:
              panic(str(e))
  
          except NameError, e:
              panic(str(e))
  
@@ -3419,13 +2650,6 @@ def sys_set_netmem_max(path, max):
          fp = open(path, 'w')
          fp.write('%d\n' %(max))
          fp.close()
          fp = open(path, 'w')
          fp.write('%d\n' %(max))
          fp.close()
-    
-    
-def sys_make_devices():
-    if not os.access('/dev/portals', os.R_OK):
-        run('mknod /dev/portals c 10 240')
-    if not os.access('/dev/obd', os.R_OK):
-        run('mknod /dev/obd c 10 241')
  
  
  # Add dir to the global PATH, if not already there.
  
  
  # Add dir to the global PATH, if not already there.
@@ -3434,7 +2658,7 @@ def add_to_path(new_dir):
      if new_dir in syspath:
          return
      os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
      if new_dir in syspath:
          return
      os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
-    
+
  def default_debug_path():
      path = '/tmp/lustre-log'
      if os.path.isdir('/r'):
  def default_debug_path():
      path = '/tmp/lustre-log'
      if os.path.isdir('/r'):
@@ -3449,7 +2673,6 @@ def default_gdb_script():
      else:
          return script
  
      else:
          return script
  
-
  DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin')
  # ensure basic elements are in the system path
  def sanitise_path():
  DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin')
  # ensure basic elements are in the system path
  def sanitise_path():
@@ -3460,6 +2683,9 @@ def sanitise_path():
  tgt_select = {}
  def init_select(args):
      # args = [service=nodeA,service2=nodeB service3=nodeC]
  tgt_select = {}
  def init_select(args):
      # args = [service=nodeA,service2=nodeB service3=nodeC]
+    # --service <service> is analagous to:
+    #     --group <service> --select <service>=<node>
+    # this is handled in doHost()
      global tgt_select
      for arg in args:
          list = string.split(arg, ',')
      global tgt_select
      for arg in args:
          list = string.split(arg, ',')
@@ -3482,6 +2708,7 @@ lconf_options = [
      ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
      ('config', "Cluster config name used for LDAP query", PARAM),
      ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
      ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
      ('config', "Cluster config name used for LDAP query", PARAM),
      ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
+    ('service', "shorthand for --group <service> --select <service>=<node>", PARAM),
      ('node',   "Load config for <nodename>", PARAM),
      ('cleanup,d', "Cleans up config. (Shutdown)"),
      ('force,f', "Forced unmounting and/or obd detach during cleanup",
      ('node',   "Load config for <nodename>", PARAM),
      ('cleanup,d', "Cleans up config. (Shutdown)"),
      ('force,f', "Forced unmounting and/or obd detach during cleanup",
@@ -3493,6 +2720,9 @@ lconf_options = [
                     another node for failover purposes. This will not
                     be a clean shutdown.""",
                 FLAG, 0),
                     another node for failover purposes. This will not
                     be a clean shutdown.""",
                 FLAG, 0),
+    ('abort_recovery',"""Used to start a service when you know recovery
+                         will not succeed.  This will skip the recovery
+                         timeout period."""),
      ('gdb', """Prints message after creating gdb module script
                      and sleeps for 5 seconds."""),
      ('noexec,n', """Prints the commands and steps that will be run for a
      ('gdb', """Prints message after creating gdb module script
                      and sleeps for 5 seconds."""),
      ('noexec,n', """Prints the commands and steps that will be run for a
@@ -3507,19 +2737,16 @@ lconf_options = [
      ('dump',  "Dump the kernel debug log to file before portals is unloaded",
                 PARAM),
      ('write_conf', "Save all the client config information on mds."),
      ('dump',  "Dump the kernel debug log to file before portals is unloaded",
                 PARAM),
      ('write_conf', "Save all the client config information on mds."),
+    ('old_conf', "Start up service even though config logs appear outdated."),
      ('record', "Write config information on mds."),
      ('record_log', "Name of config record log.", PARAM),
      ('record_device', "MDS device name that will record the config commands",
                PARAM),
      ('record', "Write config information on mds."),
      ('record_log', "Name of config record log.", PARAM),
      ('record_device', "MDS device name that will record the config commands",
                PARAM),
-    ('root_squash', "MDS squash root to appointed uid",
-              PARAM),
-    ('no_root_squash', "Don't squash root for appointed nid",
-              PARAM),
      ('minlevel', "Minimum level of services to configure/cleanup",
                   INTPARAM, 0),
      ('minlevel', "Minimum level of services to configure/cleanup",
                   INTPARAM, 0),
-    ('maxlevel', """Maximum level of services to configure/cleanup 
+    ('maxlevel', """Maximum level of services to configure/cleanup
                      Levels are aproximatly like:
                      Levels are aproximatly like:
-                            10 - netwrk
+                            10 - network
                              20 - device, ldlm
                              30 - osd, mdd
                              40 - mds, ost
                              20 - device, ldlm
                              30 - osd, mdd
                              40 - mds, ost
@@ -3533,14 +2760,21 @@ lconf_options = [
      ('upcall',  "Set both portals and lustre upcall script", PARAM),
      ('lustre_upcall', "Set lustre upcall script", PARAM),
      ('portals_upcall', "Set portals upcall script", PARAM),
      ('upcall',  "Set both portals and lustre upcall script", PARAM),
      ('lustre_upcall', "Set lustre upcall script", PARAM),
      ('portals_upcall', "Set portals upcall script", PARAM),
+    ('group_upcall', "Set supplementary group upcall program", PARAM),
      ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
      ('ptldebug', "Set the portals debug level",  PARAM),
      ('subsystem', "Set the portals debug subsystem",  PARAM),
      ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
      ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
      ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
      ('ptldebug', "Set the portals debug level",  PARAM),
      ('subsystem', "Set the portals debug subsystem",  PARAM),
      ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
      ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
+    ('allow_unprivileged_port', "Allow connections from unprivileged ports"),
+    ('clumanager', "Generate CluManager config file for this node's cluster"),
+    ('rawprimary', "For clumanager, device of the primary quorum", PARAM, "/dev/raw/raw1"),
+    ('rawsecondary', "For clumanager, device of the secondary quorum", PARAM, "/dev/raw/raw2"),
+    ('service_scripts', "For clumanager, directory containing per-service scripts", PARAM, "/etc/lustre/services"),
+    ('make_service_scripts', "Create per-service symlinks for use with clumanager"),
  # Client recovery options
      ('recover', "Recover a device"),
  # Client recovery options
      ('recover', "Recover a device"),
-    ('group', "The group of devices to configure or cleanup", PARAM),
+    ('group,g', "The group of devices to configure or cleanup", PARAM),
      ('tgt_uuid', "The failed target (required for recovery)", PARAM),
      ('client_uuid', "The failed client (required for recovery)", PARAM),
      ('conn_uuid', "The failed connection (required for recovery)", PARAM),
      ('tgt_uuid', "The failed target (required for recovery)", PARAM),
      ('client_uuid', "The failed client (required for recovery)", PARAM),
      ('conn_uuid', "The failed connection (required for recovery)", PARAM),
@@ -3548,14 +2782,17 @@ lconf_options = [
      ('inactive', """The name of an inactive service, to be ignored during
                      mounting (currently OST-only). Can be repeated.""",
                  PARAMLIST),
      ('inactive', """The name of an inactive service, to be ignored during
                      mounting (currently OST-only). Can be repeated.""",
                  PARAMLIST),
-    ]      
+    ('user_xattr', """Enable user_xattr support on MDS""", FLAG, 0),
+    ('acl', """Enable ACL support on MDS""", FLAG, 0),
+    ('quota', "Enable quota support for client file system", PARAM), 
+    ]
  
  def main():
  
  def main():
-    global lctl, config, toplustreDB, CONFIG_FILE, mod_manager
+    global lctl, config, toplustreDB, CONFIG_FILE
  
      # in the upcall this is set to SIG_IGN
      signal.signal(signal.SIGCHLD, signal.SIG_DFL)
  
      # in the upcall this is set to SIG_IGN
      signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-    
+
      cl = Lustre.Options("lconf", "config.xml", lconf_options)
      try:
          config, args = cl.parse(sys.argv[1:])
      cl = Lustre.Options("lconf", "config.xml", lconf_options)
      try:
          config, args = cl.parse(sys.argv[1:])
@@ -3578,7 +2815,7 @@ def main():
      random.seed(seed)
  
      sanitise_path()
      random.seed(seed)
  
      sanitise_path()
-    
+
      init_select(config.select)
  
      if len(args) > 0:
      init_select(config.select)
  
      if len(args) > 0:
@@ -3624,6 +2861,10 @@ def main():
          print 'see lconf --help for command summary'
          sys.exit(1)
  
          print 'see lconf --help for command summary'
          sys.exit(1)
  
+    if config.reformat and config.cleanup:
+        panic("Options \"reformat\" and \"cleanup\" are incompatible. "+
+              "Please specify only one.")
+
      toplustreDB = lustreDB
  
      ver = lustreDB.get_version()
      toplustreDB = lustreDB
  
      ver = lustreDB.get_version()
@@ -3639,7 +2880,7 @@ def main():
      else:
          if len(host) > 0:
              node_list.append(host)
      else:
          if len(host) > 0:
              node_list.append(host)
-        node_list.append('localhost')
+#        node_list.append('localhost')
  
      debug("configuring for host: ", node_list)
  
  
      debug("configuring for host: ", node_list)
  
@@ -3652,23 +2893,22 @@ def main():
      if config.lctl_dump:
          lctl.use_save_file(config.lctl_dump)
  
      if config.lctl_dump:
          lctl.use_save_file(config.lctl_dump)
  
+    if not (config.reformat or config.write_conf or config.cleanup):
+        doCheckMtime(lustreDB, node_list)
+
      if config.record:
          if not (config.record_device and config.record_log):
              panic("When recording, both --record_log and --record_device must be specified.")
          lctl.clear_log(config.record_device, config.record_log)
          lctl.record(config.record_device, config.record_log)
  
      if config.record:
          if not (config.record_device and config.record_log):
              panic("When recording, both --record_log and --record_device must be specified.")
          lctl.clear_log(config.record_device, config.record_log)
          lctl.record(config.record_device, config.record_log)
  
-    # init module manager
-    mod_manager = kmod_manager(config.lustre, config.portals)
-
-    doHost(lustreDB, node_list)
-
-    if not config.record:
-        return
-
-    lctl.end_record()
+    if config.clumanager:
+        doClumanager(lustreDB, node_list)
+    else:
+        doHost(lustreDB, node_list)
  
  
-    process_updates(lustreDB, config.record_device, config.record_log)
+    if config.record:
+        lctl.end_record()
  
  if __name__ == "__main__":
      try:
  
  if __name__ == "__main__":
      try:
@@ -3679,7 +2919,10 @@ if __name__ == "__main__":
          sys.exit(1)
      except CommandError, e:
          e.dump()
          sys.exit(1)
      except CommandError, e:
          e.dump()
-        sys.exit(e.rc)
+        rc = e.rc
+        if rc == 0:
+            rc = 1
+        sys.exit(rc)
  
      if first_cleanup_error:
          sys.exit(first_cleanup_error)
  
      if first_cleanup_error:
          sys.exit(first_cleanup_error)