From bcae4ad9487cf8eea70fd05a0da1a61639c2f70a Mon Sep 17 00:00:00 2001 From: wangdi Date: Wed, 16 Jun 2004 13:48:59 +0000 Subject: [PATCH] 1) add cmobd support in lconf and lmc 2) remove and modify some test scripts according to the new lconf and lmc --- lustre/tests/cmobd_cobd_setup.sh | 86 -------------------------- lustre/tests/cobd_test.sh | 8 +-- lustre/tests/ll_cobd_cleanup.sh | 17 ------ lustre/tests/ll_cobd_setup.sh | 21 ------- lustre/tests/uml_cobd.sh | 15 ++++- lustre/utils/lconf | 126 +++++++++++++++++++++++++++++++++++++-- lustre/utils/lmc | 95 +++++++++++++++++++++++++++-- 7 files changed, 225 insertions(+), 143 deletions(-) delete mode 100644 lustre/tests/cmobd_cobd_setup.sh delete mode 100644 lustre/tests/ll_cobd_cleanup.sh delete mode 100644 lustre/tests/ll_cobd_setup.sh diff --git a/lustre/tests/cmobd_cobd_setup.sh b/lustre/tests/cmobd_cobd_setup.sh deleted file mode 100644 index f1f203f..0000000 --- a/lustre/tests/cmobd_cobd_setup.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -LCTL=${LCTL:-"../utils/lctl"} - -CACHE_OST_UUID=${CACHE_OST_UUID:-"OST_uml_UUID"} -MASTER_OST_UUID=${MASTER_OST_UUID:-"OST_uml_2_UUID"} - -MASTER_NID_UUID=${MASTER_NID_UUID:-"NID_uml_UUID"} -MASTER_HOST=${MASTER_HOST:-"uml"} -MASTER_IP=${MASTER_IP:-"192.168.0.8"} - -MASTER_OSC=${MASTER_OSC:-"master_osc"} -MASTER_LOV=${MASTER_LOV:-"master_lov"} -MASTER_LOV_UUID=${MASTER_LOV_UUID:-"master_lov_UUID"} -MASTER_MDS_UUID=${MASTER_MDS_UUID:-"mds2_UUID"} -CACHE_MDS_UUID=${CACHE_MDS_UUID:-"mds1_UUID"} - -MASTER_MDC=${MASTER_MDC:-"master_mdc"} -MASTER_MDC_UUID=${MASTER_MDC_UUID:-"master_mdc_UUID"} -CMOBD_OST=${CMOBD:-"cmobd_ost"} -CMOBD_OST_UUID=${CMOBD_UUID:-"cmobd_ost_UUID"} -CMOBD_MDS=${CMOBD:-"cmobd_mds"} -CMOBD_MDS_UUID=${CMOBD_UUID:-"cmobd_mds_UUID"} -MASTER_OST_UP=${MASTER_OST_UP:-1} -MASTER_MDS_UP=${MASTER_MDS_UP:-1} - -echo "add uuid ${MASTER_NID_UUID} and connect to ${MASTER_IP} ..." -${LCTL} << EOF -network tcp -add_uuid ${MASTER_NID_UUID} ${MASTER_IP} tcp -EOF -echo "done!" -#send_mem 8388608 -#recv_mem 8388608 -#add_autoconn ${MASTER_IP} ${MASTER_HOST} 988 -#connect ${MASTER_IP} 988 -echo "insmod cmobd.o" -insmod "../cmobd/cmobd.o" || exit 1 -echo "reprovide gdb-friendly module information" -[ -d /r ] && ${LCTL} modules > /r/tmp/ogdb-`hostname` - - -if [ ${MASTER_OST_UP} == 1 ]; then - echo "add osc for master lov ..." -${LCTL} << EOF -newdev -attach osc ${MASTER_OSC} ${MASTER_LOV_UUID} -setup ${MASTER_OST_UUID} ${MASTER_NID_UUID} -EOF - echo "done!" - - echo "add master lov ..." -${LCTL} << EOF -newdev -attach lov ${MASTER_LOV} ${MASTER_LOV_UUID} -lov_setup ${MASTER_LOV_UUID} 1 65536 0 0 ${MASTER_OST_UUID} -EOF - echo "done!" - - echo "setup cmobd lov..." -${LCTL} << EOF -newdev -attach cmobd ${CMOBD_OST} ${CMOBD_OST_UUID} -setup ${MASTER_LOV_UUID} ${CACHE_OST_UUID} -EOF - echo "done!" -fi - - -if [ ${MASTER_MDS_UP} == 1 ]; then - echo "add mdc for master mds ..." -${LCTL} << EOF -newdev -attach mdc ${MASTER_MDC} ${MASTER_MDC_UUID} -setup ${MASTER_MDS_UUID} ${MASTER_NID_UUID} -EOF - echo "done!" - - echo "setup cmobd mdc..." -${LCTL} << EOF -newdev -attach cmobd ${CMOBD_MDS} ${CMOBD_MDS_UUID} -setup ${MASTER_MDC_UUID} ${CACHE_MDS_UUID} -EOF - echo "done!" -fi diff --git a/lustre/tests/cobd_test.sh b/lustre/tests/cobd_test.sh index a55cc76..08b9de8 100644 --- a/lustre/tests/cobd_test.sh +++ b/lustre/tests/cobd_test.sh @@ -15,11 +15,11 @@ REALMDS=${REALMDS:-$TMP/mds2-uml} CACHEOST=${CACHEOST:-$TMP/ost1-uml} REALOST=${REALOST:-$TMP/ost2-uml} -MDS_CMOBD_INDEX=${MDS_CMOBD_INDEX:-22} -OST_CMOBD_INDEX=${OST_CMOBD_INDEX:-20} +MDS_CMOBD_INDEX=${MDS_CMOBD_INDEX:-12} +OST_CMOBD_INDEX=${OST_CMOBD_INDEX:-14} -MDS_COBD_INDEX=${MDS_COBD_INDEX:-17} -OST_COBD_INDEX=${OST_COBD_INDEX:-14} +MDS_COBD_INDEX=${MDS_COBD_INDEX:-22} +OST_COBD_INDEX=${OST_COBD_INDEX:-19} if ! [ -e "$TMP/$TARBALL" ]; then diff --git a/lustre/tests/ll_cobd_cleanup.sh b/lustre/tests/ll_cobd_cleanup.sh deleted file mode 100644 index 25324ad..0000000 --- a/lustre/tests/ll_cobd_cleanup.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -#Assumption run in UML - -export PATH=`dirname $0`/../utils:$PATH - -LCONF=${LCONF:-lconf} -NAME=${NAME:-uml_cobd} -NODE=${NODE:-`hostname`} -CLEAN=${CLEAN:-"cmobd_cobd_cleanup"} -config=$NAME.xml - -echo "setup test script" -sh ${CLEAN}.sh - -echo "cleanup cmobd xml" -${LCONF} --node $NODE --cleanup $config - diff --git a/lustre/tests/ll_cobd_setup.sh b/lustre/tests/ll_cobd_setup.sh deleted file mode 100644 index ed9c5ad..0000000 --- a/lustre/tests/ll_cobd_setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -#Assumption run in UML - -export PATH=`dirname $0`/../utils:$PATH - -LCONF=${LCONF:-lconf} -NAME=${NAME:-uml_cobd} -NODE=${NODE:-`hostname`} -SETUP=${SETUP:-"cmobd_cobd_setup"} - -config=$NAME.xml -mkconfig=$NAME.sh - -sh $mkconfig $config || exit 1 - -echo "make cmobd xml" -${LCONF} --node $NODE --reformat $config - -echo "setup test script" -sh ${SETUP}.sh - diff --git a/lustre/tests/uml_cobd.sh b/lustre/tests/uml_cobd.sh index d52b5cb..7e6ee93 100644 --- a/lustre/tests/uml_cobd.sh +++ b/lustre/tests/uml_cobd.sh @@ -8,6 +8,11 @@ TMP=${TMP:-/tmp} COBD_MDS=${COBD_MDS:-"cobd1"} COBD_OST=${COBD_OST:-"cobd2"} + +CMOBD_MDS=${CMOBD_MDS:-"cmobd1"} +CMOBD_OST=${CMOBD_OST:-"cmobd2"} + + CACHE_MDS=${CACHE_MDS:-"mds1"} REAL_MDS=${REAL_MDS:-"mds2"} MDS_CACHE_DEV=$TMP/mds1-`hostname` @@ -84,9 +89,13 @@ echo "add real ost on $OSTNODE" ${LMC} -m $config --add ost --node $NODE --lov $REAL_LOV \ --fstype $BACKUP_FSTYPE --dev $OST_REAL_DEV --size $OSTSIZE || exit 21 -echo "add mds lov: $MDS_COBD" -${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST --real_obd $REAL_LOV --cache_obd $CACHE_LOV -${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS --real_obd $REAL_MDS --cache_obd $CACHE_MDS +echo "add mds lov: $COBD_MDS $COBD_OST" +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST --real_obd $REAL_LOV --cache_obd $CACHE_LOV || exit 22 +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS --real_obd $REAL_MDS --cache_obd $CACHE_MDS || exit 22 # create client config(s) +echo "add cmobd: $CMOBD_MDS $CMOBD_OST" +${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS --master_dev $MDS_REAL_DEV --cache_dev $MDS_DEV || exit 23 +${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_OST --master_dev $OST_REAL_DEV --cache_dev $OST_DEV || exit 23 + ${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre --mds $COBD_MDS --lov $COBD_OST || exit 30 diff --git a/lustre/utils/lconf b/lustre/utils/lconf index 6c2ea4d..ac5dc22 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -115,6 +115,7 @@ subsystem_names = { "ptlrouter" : (1 << 20), "cobd" : (1 << 21), "ibnal" : (1 << 22), + "cmobd" : (1 << 23), } @@ -570,7 +571,7 @@ class LCTLInterface: self.run(cmds) # create an lov - def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt, + def lov_setup(self, name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern): cmds = """ attach lov %s %s @@ -1383,8 +1384,7 @@ class LOV(Module): self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz, self.stripe_off, self.pattern, self.devlist, self.mds_name) - lctl.lov_setup(self.name, self.uuid, - self.desc_uuid, self.mds_name, self.stripe_cnt, + lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt, self.stripe_sz, self.stripe_off, self.pattern) for (osc, index, gen, active) in self.osclist: target_uuid = osc.target_uuid @@ -2152,6 +2152,116 @@ class ManagementClient(Client): Client.__init__(self, db, uuid, 'mgmt_cli', '', self_name = mgmtcli_name_for_uuid(db.getUUID()), module_dir = 'mgmt') +class VLOV(Module): + def __init__(self, db, uuid, fs_name, name_override = None, config_only = None): + Module.__init__(self, 'VLOV', db) + if name_override != None: + self.name = "lov_%s" % name_override + self.add_lustre_module('lov', 'lov') + self.stripe_sz = 65536 + self.stripe_off = 0 + self.pattern = 0 + self.stripe_cnt = 1 + self.desc_uuid = self.uuid + self.uuid = generate_client_uuid(self.name) + self.fs_name = fs_name + self.osc = get_osc(db, self.uuid, fs_name) + if not self.osc: + panic('osc not found:', self.uuid) + if config_only: + self.config_only = 1 + return + self.config_only = None + def get_uuid(self): + return self.uuid + def get_name(self): + return self.name + def prepare(self): + if not config.record and is_prepared(self.name): + return + lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt, + self.stripe_sz, self.stripe_off, self.pattern) + target_uuid = self.osc.target_uuid + try: + self.osc.active = 1 + self.osc.prepare(ignore_connect_failure=0) + except CommandError, e: + print "Error preparing OSC %s\n" % osc.uuid + raise e + lctl.lov_add_obd(self.name, self.uuid, target_uuid, 0, 1) + + def cleanup(self): + target_uuid = self.osc.target_uuid + self.osc.cleanup() + if is_prepared(self.name): + Module.cleanup(self) + if self.config_only: + panic("Can't clean up config_only LOV ", self.name) + + def load_module(self): + if self.config_only: + panic("Can't load modules for config_only LOV ", self.name) + self.osc.load_module() + Module.load_module(self) + + def cleanup_module(self): + if self.config_only: + panic("Can't cleanup modules for config_only LOV ", self.name) + Module.cleanup_module(self) + self.osc.cleanup_module() + + def correct_level(self, level, op=None): + return level + +class CMOBD(Module): + def __init__(self,db): + Module.__init__(self, 'CMOBD', db) + self.name = self.db.getName(); + self.uuid = generate_client_uuid(self.name) + self.master_uuid = self.db.get_first_ref('masterobd') + self.cache_uuid = self.db.get_first_ref('cacheobd') + self.add_lustre_module('cmobd', 'cmobd') + master_obd = self.db.lookup(self.master_uuid) + if not master_obd: + panic('master obd not found:', self.master_uuid) + cache_obd = self.db.lookup(self.cache_uuid) + if not cache_obd: + panic('cache obd not found:', self.cache_uuid) + + if master_obd.get_class() == 'ost': + self.client_uuid = generate_client_uuid(self.name) + self.master= VLOV(master_obd, self.client_uuid, self.name, + "%s_master" % (self.name)) + self.master_uuid = self.master.get_uuid() + else: + self.master = get_mdc(db, self.name, self.master_uuid) + # need to check /proc/mounts and /etc/mtab before + # formatting anything. + # FIXME: check if device is already formatted. + def prepare(self): + self.master.prepare() + if not config.record and is_prepared(self.name): + return + self.info(self.master_uuid, self.cache_uuid) + lctl.newdev("cmobd", self.name, self.uuid, + setup ="%s %s" %(self.master_uuid, + self.cache_uuid)) + + def cleanup(self): + if is_prepared(self.name): + Module.cleanup(self) + self.master.cleanup() + + def load_module(self): + self.master.load_module() + Module.load_module(self) + + def cleanup_module(self): + Module.cleanup_module(self) + self.master.cleanup_module() + + def correct_level(self, level, op=None): + return level class COBD(Module): def __init__(self, db, uuid, name, type, name_override = None): @@ -2173,8 +2283,8 @@ class COBD(Module): self.cache = LOV(cache_obd, self.cache_uuid, name, "%s_cache" % (self.name)); else: - self.real = get_mdc(db, uuid, name, self.real_uuid) - self.cache = get_mdc(db, uuid, name, self.cache_uuid) + self.real = get_mdc(db, name, self.real_uuid) + self.cache = get_mdc(db, name, self.cache_uuid) # need to check /proc/mounts and /etc/mtab before # formatting anything. # FIXME: check if device is already formatted. @@ -2437,6 +2547,8 @@ def getServiceLevel(self): ret = 40 elif type in ('lmv',): ret = 45 + elif type in ('cmobd',): + ret = 50 elif type in ('mountpoint', 'echoclient'): ret = 70 else: @@ -2474,7 +2586,7 @@ def get_osc(ost_db, uuid, fs_name): osc = OSC(ost_db, uuid, fs_name) return osc -def get_mdc(db, uuid, fs_name, mds_uuid): +def get_mdc(db, fs_name, mds_uuid): mds_db = db.lookup(mds_uuid); if not mds_db: error("no mds:", mds_uuid) @@ -2611,6 +2723,8 @@ def newService(db): n = OSD(db) elif type == 'cobd': n = COBD(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID") + elif type == 'cmobd': + n = CMOBD(db) elif type == 'mdsdev': n = MDSDEV(db) elif type == 'mountpoint': diff --git a/lustre/utils/lmc b/lustre/utils/lmc index 9504f84..476477c 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -162,6 +162,11 @@ Object creation command summary: --real_obd obd_name --cache_obd obd_name +--add cmobd + --node node_name + --master_dev obd_name + --cache_dev obd_name + --commit - Close a configuration version, and start a new one """ @@ -245,6 +250,12 @@ lmc_options = [ ('cache_obd', "Specify the cache device for the cache obd system.", PARAM), ('cobd', "Specify COBD name", PARAM), + # cmobd + ('master_dev', "Specify the master device for the cmobd system.", PARAM), + ('cache_dev', "Specify the cache device for the cmobd obd system.", PARAM), + ('cmobd', "Specify COBD name", PARAM), + + ('mgmt', "Specify management/monitoring service name.", PARAM, ""), # lmv @@ -339,6 +350,12 @@ class GenConfig: ref = self.doc.createElement(tag) ref.setAttribute("uuidref", uuid) return ref + + def dev(self, devname): + """ generate """ + tgt = self.doc.createElement('dev') + tgt.setAttribute("dev", devname) + return tgt def newService(self, tag, name, uuid): """ create a new service elmement, which requires name and uuid attributes """ @@ -419,7 +436,9 @@ class GenConfig: osd.setAttribute('osdtype', osdtype) osd.appendChild(self.ref("target", ost_uuid)) osd.appendChild(self.ref("node", node_uuid)) - if fstype: + osd.appendChild(self.dev(devname)) + + if fstype: self.addElement(osd, "fstype", fstype) if backfstype: self.addElement(osd, "backfstype", backfstype) @@ -448,6 +467,12 @@ class GenConfig: cobd.appendChild(self.ref("cacheobd",cache_uuid)) return cobd + def cmobd(self, name, uuid, real_uuid, cache_uuid): + cmobd = self.newService("cmobd", name, uuid) + cmobd.appendChild(self.ref("masterobd",real_uuid)) + cmobd.appendChild(self.ref("cacheobd",cache_uuid)) + return cmobd + def ost(self, name, uuid, osd_uuid, group=""): ost = self.newService("ost", name, uuid) ost.appendChild(self.ref("active", osd_uuid)) @@ -518,6 +543,8 @@ class GenConfig: mdd.appendChild(self.ref("node", node_uuid)) mdd.appendChild(self.ref("target", mds_uuid)) + mdd.appendChild(self.dev(devname)) + if lmv_uuid: mdd.appendChild(self.ref("lmv", lmv_uuid)) self.addElement(mdd, "lmv", lmv_uuid) @@ -687,18 +714,19 @@ def lov_add_obd(gen, lustre, lov, osc_uuid, options): addUpdate(gen, lustre, gen.add(getUUID(lov), osc_uuid, str(lov_index), str(gener))) return - lov.appendChild(gen.lov_tgt(osc_uuid, str(lov_index), '1')) + gener = 1 + lov.appendChild(gen.lov_tgt(osc_uuid, str(lov_index), '1')) addUpdate(gen, lustre, gen.add(getUUID(lov), osc_uuid, str(lov_index), str(gener))) return index = -1 - for tgt in lustre.getElementsByTagName('lov_tgt'): + for tgt in lov.getElementsByTagName('lov_tgt'): uuidref = tgt.getAttribute('uuidref') tmp = int(tgt.getAttribute('index')) if tmp != index + 1: error('malformed xml: LOV targets are not ordered; found index '+str(tmp)+', expected '+str(index + 1)+'.') - index = tmp + index = tmp lov.appendChild(gen.lov_tgt(osc_uuid, str(index + 1), '1')) addUpdate(gen, lustre, gen.add(getUUID(lov), osc_uuid, str(index + 1), '1')) @@ -769,6 +797,40 @@ def node_add_profile(gen, node, ref, uuid): profile.appendChild(gen.ref(ref, uuid)) return 1 +# ensure that uuid is not already in the profile +# return true if uuid is added +def node_found_target_by_dev(gen, lustre, node, devname): + refname = "%s_ref" % "profile" + ret = node.getElementsByTagName(refname) + if not ret: + error('node has no profile ref:', node) + prof_uuid = ret[0].getAttribute('uuidref') + profile = lookup(node.parentNode, prof_uuid) + if not profile: + error("no profile found:", prof_uuid) + + osd_list = lustre.getElementsByTagName('osd') + + for osd in osd_list: + obd_dev = osd.getElementsByTagName('dev') + if obd_dev and obd_dev[0].getAttribute('dev') == devname: + for ost in lustre.getElementsByTagName('ost'): + active_ret = ost.getElementsByTagName('active_ref') + if active_ret[0].getAttribute('uuidref') == osd.getAttribute('uuid'): + return ost.getAttribute('uuid') + + mdsdev_list = lustre.getElementsByTagName('mdsdev') + + for mdsdev in mdsdev_list: + obd_dev = mdsdev.getElementsByTagName('dev') + if obd_dev and obd_dev[0].getAttribute('dev') == devname: + for mds in lustre.getElementsByTagName('mds'): + active_ret = mds.getElementsByTagName('active_ref') + if active_ret[0].getAttribute('uuidref') == mdsdev.getAttribute('uuid'): + return mds.getAttribute('uuid') + + return "" + def get_attr(dom_node, attr, default=""): v = dom_node.getAttribute(attr) if v: @@ -1081,6 +1143,25 @@ def del_ost(gen, lustre, options): # delete the OST lustre.removeChild(ost) +def add_cmobd(gen, lustre, options): + node_name = get_option(options, 'node') + name = get_option(options, 'cmobd') + uuid = new_uuid(name) + + real_name = get_option(options, 'master_dev') + cache_name = get_option(options, 'cache_dev') + + node = findByName(lustre, node_name, "node") + node_add_profile(gen, node, "cmobd", uuid) + real_uuid = node_found_target_by_dev(gen, lustre, node, real_name) + cache_uuid = node_found_target_by_dev(gen, lustre, node, cache_name) + if not real_uuid: + panic("add_cmobd", "can not find real_uuid") + if not cache_uuid: + panic("add_cmobd", "can not find cache_uuid") + cmobd = gen.cmobd(name, uuid, real_uuid, cache_uuid) + lustre.appendChild(cmobd) + def add_cobd(gen, lustre, options): node_name = get_option(options, 'node') name = get_option(options, 'cobd') @@ -1094,7 +1175,7 @@ def add_cobd(gen, lustre, options): if real_uuid: node = lookup(lustre, real_uuid) - rets = node.getElementsByTagName('obd_ref') + rets = node.getElementsByTagName('lov_tgt') for ret in rets: ost_uuid = ret.getAttribute('uuidref') ost_node = lookup(lustre, ost_uuid) @@ -1106,7 +1187,7 @@ def add_cobd(gen, lustre, options): if cache_uuid: node = lookup(lustre, cache_uuid) - rets = node.getElementsByTagName('obd_ref') + rets = node.getElementsByTagName('lov_tgt') for ret in rets: ost_uuid = ret.getAttribute('uuidref') ost_node = lookup(lustre, ost_uuid) @@ -1464,6 +1545,8 @@ def add(devtype, gen, lustre, options): add_echo_client(gen, lustre, options) elif devtype == 'cobd': add_cobd(gen, lustre, options) + elif devtype == 'cmobd': + add_cmobd(gen, lustre, options) elif devtype == 'mgmt': add_mgmt(gen, lustre, options) elif devtype == 'lmv': -- 1.8.3.1