--- /dev/null
+#!/usr/bin/python2
+"""
+This module implements lustre-specific IO- and network-tests.
+It is based on the 'obdfilter-survey'-script distributed with lustre-iokit.
+
+To use it as a library, the caller should first create a set of
+EchoClient-object. The EchoClient-class will automatically create the
+echo_client-device, and set it up to communicate with the device
+given as the target to the EchoClient-constructor. See main() for
+an example of how to set up EchoClient-objects and the objects it
+depends on.
+
+Next, run ParallelTestBRW to run benchmarks in parallel over all
+the EchoClients with a specific number of threads++.
+
+ParallelTestBRW returns a list of ParallelTestBRWResult-objects
+(one for eacy type of test ('w' and 'r') performed).
+See the documentation for ParallelTestBRWResult for how to extract
+the data from this object.
+
+Some notes about the implementation:
+The core-functionality is implemented as python-classes wrapping lustre-devices
+such as obdecho-, osc-, and echo_client-devices. The constructors for these
+classes automatically create the lustre-device, and the destructor removes the
+devices. High-level devices keep references to low-level devices, ensuring that
+the low-level devices are not removed as long as they are in use. The
+garbage-collector will clean everything up in the right order. However, there
+are two corner-cases that users of the library must be awere of:
+
+1. You can not create to lustre-devices of the same type with the same name on
+the same node at the same time. Replacing one object with a conflicting object
+like this:
+
+foo = OBDEcho("nodename", "test_obdecho")
+foo = OBDEcho("nodename", "test_obdecho")
+
+will fail because the second obdecho-object's constructor will run before the old
+object has been removed. To replace an object with a conflicting new object the
+fist one has to explisitly be removed first:
+
+foo = OBDEcho("nodename", "test_obdecho")
+del foo
+foo = OBDEcho("nodename", "test_obdecho")
+
+
+2. When python exists it will remove all remaining objects without following
+the dependency-rules between objects. This may cause lustre-devices to not
+be removed properly. Make sure to delete all references to the lustre-device
+objects _before_ exiting python to make sure this doesn't happen.
+
+
+
+Copyright (c) 2005 Scali AS. All Rights Reserved.
+"""
+
+import string
+import time
+import random
+import re
+import logging
+import os
+import popen2
+
+
+
+# Classes that implement remote execution using different tools/protocols:
+# These should subclass Popen3, and implement the same interface.
+
+class scashPopen3(popen2.Popen3):
+ """
+ Implement the same functionality as popen2.Popen3, but for a
+ remote machine.
+ """
+ def __init__(self, node, cmd, *args, **kwargs):
+ """
+ As popen2.Popen3, except:
+ @node - hostname where to execute cmd
+ @cmd - the command to execute. (Needs to be a string!)
+ """
+ cmd = ["scash", "-n", node, cmd]
+ popen2.Popen3.__init__(self, cmd, *args, **kwargs)
+
+class sshPopen3(popen2.Popen3):
+ """
+ Implement the same functionality as popen2.Popen3, but for a
+ remote machine.
+ """
+ def __init__(self, node, cmd, *args, **kwargs):
+ """
+ As popen2.Popen3, except:
+ @node - hostname where to execute cmd
+ @cmd - the command to execute. (Needs to be a string!)
+ """
+ cmd = ["ssh", node, cmd]
+ popen2.Popen3.__init__(self, cmd, *args, **kwargs)
+
+
+# Select remote execution tool/protocol based on what is actually available:
+if os.path.isfile("/opt/scali/bin/scash"):
+ remotePopen3 = scashPopen3
+elif os.path.isfile("/usr/bin/ssh"):
+ remotePopen3 = sshPopen3
+else:
+ raise Exception("No remote-execution environment found!")
+
+
+def remoteCommand(node, command):
+ """
+ Run an external command, and return the output as a list of strings
+ (one string per line). Raise an exception if the command fails
+ (returns non-zero exit-code).
+ @node - nodename where to run the command
+ @command - the command to run
+ """
+ remote = remotePopen3(node, command, True)
+ exit_code = remote.wait()
+ if exit_code != 0 :
+ raise Exception("Remote command %s failed with exit-code: %d" %
+ (repr(command), exit_code))
+ return remote.fromchild.readlines()
+
+def genUUID():
+ """
+ Generate a random UUID
+ """
+ r = random.Random(time.time())
+ return "%04x%04x-%04x-%04x-%04x-%04x%04x%04x" % (r.randint(0,16**4), r.randint(0,16**4),
+ r.randint(0,16**4), r.randint(0,16**4), r.randint(0,16**4),
+ r.randint(0,16**4), r.randint(0,16**4), r.randint(0,16**4))
+
+class KernelModule:
+ """
+ Object to keep track of the usage of a kernel-module, and unload it when
+ it's no longer needed. The constructor will check if the module is already
+ loaded. If it is, the use_count will be preset to 1 and the module will never
+ be automatically unloaded. (Assuming no object will cal decUse without first
+ having called incUse)
+ """
+ def __init__(self, node, name):
+ """
+ KernelModule constructor.
+ Does _not_ increase the usage-counter or load the module!
+ @name - the name of the kernel-module
+ """
+ self.node = node
+ self.name = name
+ self.use_count = self.__isLoaded()
+ def __isLoaded(self):
+ """
+ Check if the module is currently loaded
+ """
+ for line in remoteCommand(self.node, "/sbin/lsmod"):
+ if line.split()[0] == self.name:
+ return 1
+ return 0
+ def __load(self):
+ """
+ Load the module now
+ Don't call this directly - call incUse.
+ """
+ remoteCommand(self.node, "modprobe %s" % self.name)
+ def __unload(self):
+ """
+ Unload the module now.
+ Don't call this directly - call decUse.
+ """
+ remoteCommand(self.node, "rmmod obdecho")
+ def incUse(self):
+ """
+ Call this method before using the module
+ """
+ self.use_count += 1
+ if self.use_count == 1:
+ self.__load()
+ def decUse(self):
+ """
+ Call this method when you're done using the module
+ """
+ self.use_count -= 1
+ if self.use_count == 0:
+ self.__unload()
+
+
+class KernelModules:
+ """
+ Class to keep track of multiple KernelModule-objects
+ for multiple kernel-modules on multiple nodes.
+ """
+ def __init__(self):
+ # The KernelModule-objects are stored in self.data
+ # The key in self.data is the nodename. The value is a new
+ # new dictionary with module-names as keys and KernelModule
+ # objects as values.
+ self.data = {}
+ def getKernelModule(self, nodename, modulename):
+ """
+ Lookup (or create) a KernelModule object
+ @nodename - the node where the kernel-module should be loaded
+ @modulename - the name of the kernel-module
+ """
+ # Create the object if it's not already in self.data:
+ if not self.data.has_key(nodename):
+ self.data[nodename] = {}
+ if not self.data[nodename].has_key(modulename):
+ self.data[nodename][modulename] = KernelModule(nodename, modulename)
+ # And then return it:
+ return self.data[nodename][modulename]
+
+# This global object is used to keep track of all the loaded kernel-modules:
+modules = KernelModules()
+
+def lctl(node, commands):
+ """
+ Run a set of lctl-commands
+ @node - node where to run the commands
+ @commands - list of commands
+ Returns the output from lctl as a list of strings (one string per line)
+ """
+ # Encapsulate in quotes:
+ commands = string.join(commands, '\n')
+ log = logging.getLogger("lctl")
+ log.debug("lctl: %s" % repr(commands))
+ return remoteCommand(node, 'echo -e "%s" | lctl' % commands)
+
+def find_device(node, search_type, search_name):
+ """
+ Find the devicenumber for a device
+ @ node - the node where the device lives
+ @ search_type - the device-type to search for
+ @ search_name - the devine-name to search for
+ Returns the device-number (int)
+ """
+
+ for dev in lctl(node, ['device_list']):
+ device_id, device_state, device_type, device_name, uuid, refcnt = dev.split()
+ if device_type == search_type and device_name == search_name:
+ return int(device_id)
+ raise ValueError("device not found: %s:%s" % (search_type, search_name))
+
+
+class OBDEcho:
+ """
+ Create a obdecho-device (A device that can simulate a ost)
+ """
+ def __init__(self, node, name):
+ """
+ The constructor will create the device
+ @node - the node where to run the obdecho-device
+ @name - the name of the new device
+ """
+ self.node = node
+ self.name = name
+ self.uuid = genUUID()
+ self.module = modules.getKernelModule(self.node, "obdecho")
+ self.module.incUse()
+ lctl(self.node, ['attach obdecho %s %s' % (self.name, self.uuid), 'setup n'])
+ def __del__(self):
+ """
+ The destructor will remove the device
+ """
+ lctl(self.node, ['cfg_device %s' % self.name, 'cleanup', 'detach'])
+ self.module.decUse()
+
+
+class ExistingOSC:
+ """
+ Class to represent an existing osc-device
+ The object is device is not manipulated in any way - this class
+ is just used to keep refer to the device
+ """
+ def __init__(self, node, name):
+ """
+ Create a reference to the device
+ @node - the node where the device lives
+ @name - the name of the device
+ """
+ self.node = node
+ self.name = name
+
+class OSC:
+ """
+ Create a osc-device (A device that connects to a remote ost/obdecho-device
+ and looks like a local obdfilter.
+ """
+ def __init__(self, node, name, ost):
+ """
+ Create the device
+ @node - the node where to run the OSC
+ @name - the name of the new device
+ @ost - the object that the osc should be connected to. This should
+ be an OBDEcho-object
+ """
+ self.node = node
+ self.name = name
+ self.ost = ost
+ self.module = modules.getKernelModule(self.node, "obdecho")
+ self.module.incUse()
+ self.uuid = genUUID()
+ # FIXME: "NID_%s_UUID" should probably not be hardcoded? Retrieve uuid from node-object?
+ lctl(self.node, ['attach osc %s %s' % (self.name, self.uuid), 'setup %s "NID_%s_UUID"' % (self.ost.uuid, self.ost.node)])
+ def __del__(self):
+ """
+ The destructor will remove the device
+ """
+ lctl(self.node, ['cfg_device %s' % self.name, 'cleanup', 'detach'])
+ self.module.decUse()
+
+class ExistingOBDFilter:
+ """
+ Class to represent an existing obdfilter-device
+ The object is device is not manipulated in any way - this class
+ is just used to keep refer to the device
+ """
+ def __init__(self, node, name):
+ """
+ Create a reference to the device
+ @node - the node where the device lives
+ @name - the name of the device
+ """
+ self.node = node
+ self.name = name
+
+class EchoClient:
+ """
+ Class wrapping echo_client functionality
+ """
+ def __init__(self, node, name, target):
+ """
+ Create a new echo_client
+ @node - the node to run the echo_client on
+ @name - the name of the new echo_client
+ @target - The obdfilter / osc device to connect to. This should
+ be an OSC, ExistingOSC or ExistingOBDFilter-object on the same node.
+ """
+ self.node = node
+ self.name = name
+ self.target = target
+ self.objects = [] # List of objects that have been created and not yet destroyed.
+ self.log = logging.getLogger("EchoClient")
+ self.module = modules.getKernelModule(self.node, "obdecho")
+ self.module.incUse()
+ self.uuid = genUUID()
+ lctl(self.node, ['attach echo_client %s %s' % (self.name, self.uuid), 'setup %s' % self.target.name])
+ self.devicenum = find_device(self.node, 'echo_client', self.name)
+ self.log.debug("EchoClient created: %s" % self.name)
+
+ def __del__(self):
+ """
+ Remove the echo_client, and unload the obdecho module if it is no longer in use
+ Destroy all objects that have been created.
+ """
+ self.log.debug("EchoClient destructor: destroying objects")
+ self.destroyObjects(self.objects[:])
+ self.log.debug("EchoClient destructor: detach echo_client:")
+ lctl(self.node, ['cfg_device %s' % self.name, 'cleanup', 'detach'])
+ self.log.debug("EchoClient destructor: Unload modules:")
+ self.module.decUse()
+ self.log.debug("EchoClient destructor: Done")
+
+ def createObjects(self, num):
+ """
+ Create new objects on this device
+ @num - the number of devices to create
+ Returns a list of object-ids.
+ """
+ oids = []
+ line = lctl(self.node, ['device %d' % self.devicenum, 'create %d' % num])
+ if line[0].strip() != 'create: %d objects' % num:
+ raise Exception("Invalid output from lctl(2): %s" % repr(line[1]))
+ pattern=re.compile('create: #(.*) is object id 0x(.*)')
+ for line in line[1:]:
+ i, oid = pattern.match(line).groups()
+ if int(i) != len(oids)+1:
+ raise Exception("Expected to find object nr %d - found object nr %d:" % ( len(oids)+1, int(i)))
+ oids.append(long(oid, 16))
+ self.objects += oids
+ return oids
+
+ def destroyObjects(self, objects):
+ """
+ Destroy a set of objects
+ @objects - list of object ids
+ """
+ for oid in objects:
+ lctl(self.node, ['device %d' % self.devicenum, 'destroy %d' % oid])
+ self.objects.remove(oid)
+
+ def startTestBRW(self, oid, threads=1, num=1, test='w', pages=1):
+ """
+ Start an test_brw, and return a remotePopen3-object to the test-process
+ Do <num> bulk read/writes on OST object <objid> (<npages> per I/O).
+ @oid - objectid for the first object to use.
+ (each thread will use one object)
+ @threads - number of threads to use
+ @num - number of io-operations to perform
+ @test - what test to perform ('w' or 'r', for write or read-tests)
+ @pages - number of pages to use in each io-request. (4KB on ia32)
+ """
+ cmd = 'lctl --threads %d q %d test_brw %d %s q %d %d' % \
+ (threads, self.devicenum, num, test, pages, oid)
+
+ self.log.debug("startTestBRW: %s:%s" % (self.node, cmd))
+ remote = remotePopen3(self.node, cmd, True)
+ return remote
+
+ def testBRW(self, oid, threads=1, num=1, test='w', pages=1):
+ """
+ Do <num> bulk read/writes on OST object <objid> (<npages> per I/O).
+ @oid - objectid for the first object to use.
+ (each thread will use one object)
+ @threads - number of threads to use
+ @num - number of io-operations to perform
+ @test - what test to perform ('w' or 'r', for write or read-tests)
+ @pages - number of pages to use in each io-request. (4KB on ia32)
+ """
+ test = self.startTestBRW(oid, threads, num, test, pages)
+ exit_code = test.wait()
+ if exit_code != 0:
+ raise Exception("test_brw failed with exitcode %d." % exit_code)
+
+class ParallelTestBRWResult:
+ """
+ Class to hold result from ParallelTestBRW
+ """
+ def __init__(self, threads, num, testtype, pages, pagesize, numclients):
+ """
+ Prepare the result-object with the constants for the test
+ threads -- number of threads (per client)
+ num -- number of io-operations for each thread
+ testtype -- what kind of test ('w' for write-test or 'r' for read-test)
+ pages -- number of pages in each request
+ pagesize -- pagesize (Assumes same page-size accross all clients)
+ numclients -- number of clients used in the tests
+ """
+ self.threads = threads
+ self.num = num
+ self.testtype = testtype
+ self.pages = pages
+ self.pagesize = pagesize
+ self.numclients = numclients
+ self.starttimes = {} # clientid to starttime mapping
+ self.finishtimes = {} # clientid to finishtime mapping
+ self.exitcodes = {} # clientid to exit-code mapping
+ self.runtimes = {} # clientid to runtime mapping
+ self.stdout = {} # clientid to output mapping
+ self.stderr = {} # clientid to errors mapping
+ def registerStart(self, clientid):
+ """
+ Register that this client is about to start
+ clientid -- the id of the client
+ """
+ self.starttimes[clientid] = time.time()
+ def registerFinish(self, clientid, exitcode, stdout, stderr):
+ """
+ Register that this client just finished
+ clientid -- the id of the client
+ exitcode -- the exitcode of this test
+ stdout -- the output from the test
+ stderr -- the errors from the test
+ """
+ self.finishtimes[clientid] = time.time()
+ self.exitcodes[clientid] = exitcode
+ self.stdout[clientid] = stdout
+ self.stderr[clientid] = stderr
+ self.runtimes[clientid] = self.finishtimes[clientid] - self.starttimes[clientid]
+ def getTestType(self):
+ """
+ Return the name of the test-type ('w' for write-tests and 'r' for read-tests)
+ """
+ return self.testtype
+ def verifyExitCodes(self):
+ """
+ Verify that all tests finished successfully. Raise exception if they didn't.
+ """
+ if self.exitcodes.values().count(0) != self.numclients:
+ raise Exception("test_brw failed!")
+ def getTotalTime(self):
+ """
+ Return the number of seconds used for the test
+ """
+ return max(self.finishtimes.values()) - min(self.starttimes.values())
+ def getTotalSize(self):
+ """
+ Return total amount of data transfered (in KB)
+ """
+ return self.numclients * self.num * self.pages * self.threads * self.pagesize
+ def getTotalBandwidth(self):
+ """
+ Return the total bandwidth for the test
+ """
+ return self.getTotalSize() / self.getTotalTime()
+ def getMaxBandwidth(self):
+ """
+ Return the bandwidth of the fastest OST
+ """
+ time = min(self.runtimes.values())
+ return self.num * self.pages * self.threads * self.pagesize / time
+ def getMinBandwidth(self):
+ """
+ Return the bandwidth of the fastest OST
+ """
+ time = max(self.runtimes.values())
+ return self.num * self.pages * self.threads * self.pagesize / time
+
+
+
+def ParallelTestBRW(echo_clients, threads=1, size=100, tests=('w', 'r'), rsz=1024, pagesize=4):
+ """
+ Run a test_brw in parallel on a set of echo_clients
+ @echo_client -- list of EchoClient-objects to run tests on
+ @threads -- number of threads to use per client
+ @size -- amount of data to transfer for each thread (MB)
+ @test -- list of tests to perform ('w' or 'r', for write or read-tests)
+ @rsz -- Amount of data (in KB) for each request. Default, 1024.
+ @pagesize - Size of each page (KB)
+ """
+ pages = rsz / pagesize
+ num = size * 1024 / rsz / threads
+ # Create objects:
+ objects = {}
+ for client in echo_clients:
+ objects[client] = client.createObjects(threads)
+ # Verify if the objectids are consequative:
+ for i in range(len(objects[client])-1):
+ if objects[client][i+1] != objects[client][i] + 1:
+ raise Exception("Non-consequative objectids on client %s: %s" % (client, objects[client]))
+ # Run tests:
+ results = []
+ for test in tests:
+ result = ParallelTestBRWResult(threads, num, test, pages, pagesize, len(echo_clients))
+ pids = {} # pid to clientid mapping
+ remotes = {} # clientid to RemotePopen3-objects
+ # Start tests:
+ clientid = 0
+ for client in echo_clients:
+ first_obj = objects[client][0]
+ result.registerStart(clientid)
+ remote = client.startTestBRW(first_obj, threads, num, test, pages)
+ remotes[clientid] = remote
+ pids[remote.pid] = clientid
+ clientid += 1
+ # Wait for tests to finish:
+ while pids:
+ pid, status = os.wait()
+ clientid = pids[pid]
+ remote = remotes[clientid]
+ # Workaround for leak in popen2, see patch #816059 at python.sf.net:
+ popen2._active.remove(remote)
+ result.registerFinish(clientid, status, remote.fromchild.read(), remote.childerr.read())
+ del pids[pid]
+ results.append(result)
+ # Clean up objects:
+ for client in echo_clients:
+ client.destroyObjects(objects[client])
+ return results
+
+
+
+def timeit(func, *args, **kwargs):
+ """
+ Helper-function to easily time the execution of a function.
+ @func - the function to run
+ @*args - possitional arguments
+ @**kwargs - keyword arguments
+ Returns the number of seconds used executing the function
+
+ Example:
+ timeit(max, 1, 2, 5, 2) - will time how long it takes to run max(1,2,5,2)
+ """
+ t1 = time.time()
+ func(*args, **kwargs)
+ t2 = time.time()
+ return t2-t1
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff