From d36f2a1170ea911375bcc6fd2cde1667f98f1b92 Mon Sep 17 00:00:00 2001 From: yury Date: Sun, 13 Jun 2004 10:24:38 +0000 Subject: [PATCH] Added the following: - lmv_iocontrol() which currently passes all commands to underlying mds's. - some stuff in lproc_lmv.c, which shows now number of obds, number of active obds, etc. - some cleanups, checks and error messages in lmv_obd.c. --- lustre/lmv/lmv_obd.c | 60 ++++++++++++++++++----- lustre/lmv/lproc_lmv.c | 121 ++++++++++++++++++++++++++++++++++++++++++++--- lustre/tests/failover.sh | 88 ++++++++++++++++++++++++++++++++++ lustre/tests/upcall | 12 +++++ 4 files changed, 263 insertions(+), 18 deletions(-) create mode 100755 lustre/tests/failover.sh create mode 100755 lustre/tests/upcall diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index cbad914..cfb5787 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -78,10 +78,8 @@ static int lmv_set_mdc_active(struct lmv_obd *lmv, struct obd_uuid *uuid, GOTO(out, rc = -EINVAL); obd = class_exp2obd(tgt->ltd_exp); - if (obd == NULL) { - /* This can happen if OST failure races with node shutdown */ + if (obd == NULL) GOTO(out, rc = -ENOTCONN); - } CDEBUG(D_INFO, "Found OBD %s=%s device %d (%p) type %s at LMV idx %d\n", obd->obd_name, obd->obd_uuid.uuid, obd->obd_minor, obd, @@ -234,8 +232,9 @@ int lmv_check_connect(struct obd_device *obd) { lmv->connected = 1; cluuid = &lmv->cluuid; exp = lmv->exp; + CDEBUG(D_OTHER, "time to connect %s to %s\n", - cluuid->uuid, obd->obd_name); + cluuid->uuid, obd->obd_name); for (i = 0, tgts = lmv->tgts; i < lmv->desc.ld_tgt_count; i++, tgts++) { struct obd_device *tgt_obd; @@ -362,14 +361,49 @@ static int lmv_disconnect(struct obd_export *exp, int flags) lmv->tgts[i].ltd_exp = NULL; } - out_local: - /* FIXME: cleanup here */ +out_local: + /* This is the case when no real connection is established by + * lmv_check_connect(). */ if (!lmv->connected) class_export_put(exp); rc = class_disconnect(exp, 0); RETURN(rc); } +static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, + int len, void *karg, void *uarg) +{ + struct obd_device *obddev = class_exp2obd(exp); + struct lmv_obd *lmv = &obddev->u.lmv; + int i, rc = 0, set = 0; + + ENTRY; + + if (lmv->desc.ld_tgt_count == 0) + RETURN(-ENOTTY); + + for (i = 0; i < lmv->desc.ld_tgt_count; i++) { + int err; + + err = obd_iocontrol(cmd, lmv->tgts[i].ltd_exp, + len, karg, uarg); + if (err) { + if (lmv->tgts[i].active) { + CERROR("error: iocontrol MDC %s on MDT" + "idx %d: err = %d\n", + lmv->tgts[i].uuid.uuid, i, err); + if (!rc) + rc = err; + } + } else + set = 1; + } + if (!set && !rc) + rc = -EIO; + + RETURN(rc); +} + static int lmv_setup(struct obd_device *obd, obd_count len, void *buf) { struct lustre_cfg *lcfg = buf; @@ -418,8 +452,9 @@ static int lmv_setup(struct obd_device *obd, obd_count len, void *buf) for (i = 0, tgts = lmv->tgts; i < desc->ld_tgt_count; i++, tgts++) tgts->uuid = uuids[i]; - lmv->max_easize = sizeof(struct ll_fid) * desc->ld_tgt_count - + sizeof(struct mea); + lmv->max_easize = sizeof(struct ll_fid) * + desc->ld_tgt_count + sizeof(struct mea); + lmv->max_cookiesize = 0; RETURN(rc); @@ -1163,8 +1198,8 @@ int lmv_readpage(struct obd_export *exp, struct ll_fid *mdc_fid, RETURN(rc); } -int lmv_unlink_slaves(struct obd_export *exp, - struct mdc_op_data *data, struct ptlrpc_request **req) +int lmv_unlink_slaves(struct obd_export *exp, struct mdc_op_data *data, + struct ptlrpc_request **req) { struct obd_device *obd = exp->exp_obd; struct lmv_obd *lmv = &obd->u.lmv; @@ -1306,7 +1341,7 @@ int lmv_obd_create_single(struct obd_export *exp, struct obdo *oa, * to be called from MDS only */ int lmv_obd_create(struct obd_export *exp, struct obdo *oa, - struct lov_stripe_md **ea, struct obd_trans_info *oti) + struct lov_stripe_md **ea, struct obd_trans_info *oti) { struct obd_device *obd = exp->exp_obd; struct lmv_obd *lmv = &obd->u.lmv; @@ -1382,7 +1417,7 @@ int lmv_obd_create(struct obd_export *exp, struct obdo *oa, } static int lmv_get_info(struct obd_export *exp, __u32 keylen, - void *key, __u32 *vallen, void *val) + void *key, __u32 *vallen, void *val) { struct obd_device *obd; struct lmv_obd *lmv; @@ -1567,6 +1602,7 @@ struct obd_ops lmv_obd_ops = { .o_brw = lmv_brw, .o_init_ea_size = lmv_init_ea_size, .o_notify = lmv_notify, + .o_iocontrol = lmv_iocontrol, }; struct md_ops lmv_md_ops = { diff --git a/lustre/lmv/lproc_lmv.c b/lustre/lmv/lproc_lmv.c index 7867d7e..3c3bd76 100644 --- a/lustre/lmv/lproc_lmv.c +++ b/lustre/lmv/lproc_lmv.c @@ -33,10 +33,119 @@ static struct lprocfs_vars lprocfs_module_vars[] = { {0} }; static struct lprocfs_vars lprocfs_obd_vars[] = { {0} }; #else -/* FIXME: here should be definition of lprocfs_module_vars and - lprocfs_obd_vars. */ -static struct lprocfs_vars lprocfs_module_vars[] = { {0} }; -static struct lprocfs_vars lprocfs_obd_vars[] = { {0} }; -#endif -LPROCFS_INIT_VARS(lmv, lprocfs_module_vars, lprocfs_obd_vars) +static int lmv_rd_numobd(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + struct obd_device *dev = (struct obd_device*)data; + struct lmv_desc *desc; + + LASSERT(dev != NULL); + desc = &dev->u.lmv.desc; + *eof = 1; + return snprintf(page, count, "%u\n", desc->ld_tgt_count); + +} + +static int lmv_rd_activeobd(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + struct obd_device* dev = (struct obd_device*)data; + struct lmv_desc *desc; + + LASSERT(dev != NULL); + desc = &dev->u.lmv.desc; + *eof = 1; + return snprintf(page, count, "%u\n", desc->ld_active_tgt_count); +} + +static int lmv_rd_desc_uuid(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + struct obd_device *dev = (struct obd_device*) data; + struct lmv_obd *lmv; + + LASSERT(dev != NULL); + lmv = &dev->u.lmv; + *eof = 1; + return snprintf(page, count, "%s\n", lmv->desc.ld_uuid.uuid); +} + +static void *lmv_tgt_seq_start(struct seq_file *p, loff_t *pos) +{ + struct obd_device *dev = p->private; + struct lmv_obd *lmv = &dev->u.lmv; + + return (*pos >= lmv->desc.ld_tgt_count) ? NULL : &(lmv->tgts[*pos]); + +} +static void lmv_tgt_seq_stop(struct seq_file *p, void *v) +{ + return; +} + +static void *lmv_tgt_seq_next(struct seq_file *p, void *v, loff_t *pos) +{ + struct obd_device *dev = p->private; + struct lmv_obd *lmv = &dev->u.lmv; + + ++*pos; + return (*pos >=lmv->desc.ld_tgt_count) ? NULL : &(lmv->tgts[*pos]); +} + +static int lmv_tgt_seq_show(struct seq_file *p, void *v) +{ + struct lmv_tgt_desc *tgt = v; + struct obd_device *dev = p->private; + struct lmv_obd *lmv = &dev->u.lmv; + int idx = tgt - &(lmv->tgts[0]); + + return seq_printf(p, "%d: %s %sACTIVE\n", idx, tgt->uuid.uuid, + tgt->active ? "" : "IN"); +} + +struct seq_operations lmv_tgt_sops = { + .start = lmv_tgt_seq_start, + .stop = lmv_tgt_seq_stop, + .next = lmv_tgt_seq_next, + .show = lmv_tgt_seq_show, +}; + +static int lmv_target_seq_open(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *dp = PDE(inode); + struct seq_file *seq; + int rc = seq_open(file, &lmv_tgt_sops); + + if (rc) + return rc; + + seq = file->private_data; + seq->private = dp->data; + + return 0; +} + +struct lprocfs_vars lprocfs_obd_vars[] = { + { "numobd", lmv_rd_numobd, 0, 0 }, + { "activeobd", lmv_rd_activeobd, 0, 0 }, + { "uuid", lprocfs_rd_uuid, 0, 0 }, + { "desc_uuid", lmv_rd_desc_uuid, 0, 0 }, + { 0 } +}; + +static struct lprocfs_vars lprocfs_module_vars[] = { + { "num_refs", lprocfs_rd_numrefs, 0, 0 }, + { 0 } +}; + +struct file_operations lmv_proc_target_fops = { + .owner = THIS_MODULE, + .open = lmv_target_seq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +#endif /* LPROCFS */ +LPROCFS_INIT_VARS(lmv, lprocfs_module_vars, lprocfs_obd_vars) diff --git a/lustre/tests/failover.sh b/lustre/tests/failover.sh new file mode 100755 index 0000000..94595e3 --- /dev/null +++ b/lustre/tests/failover.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -e + +export PATH=`dirname $0`/../utils:$PATH + +config=${1:-lmv.xml} + +LMC=${LMC:-lmc} +TMP=${TMP:-/r/tmp} + +MDSSIZE=${MDSSIZE:-100000} +FSTYPE=${FSTYPE:-ext3} +MDSCOUNT=${MDSCOUNT:-2} +NODECOUNT=${NODECOUNT:-3} + +OSTDEV=${OSTDEV:-$TMP/ost1-`hostname`} +OSTSIZE=${OSTSIZE:-200000} + +# 1 to config an echo client instead of llite +ECHO_CLIENT=${ECHO_CLIENT:-} + +STRIPE_BYTES=65536 +STRIPES_PER_OBJ=0 + +MOUNT=${MOUNT:-/mnt/lustre} + +# specific journal size for the ost, in MB +JSIZE=${JSIZE:-0} +JARG="" +[ "$JSIZE" -gt 0 ] && JARG="--journal_size $JSIZE" + +rm -f $config + +upcall="/r/home/umka/work/cfs/lustre/tests/upcall" + +# configuring nodes +nodes_with_client=$NODECOUNT +if test $NODECOUNT -le 2; then + let nodes_with_client=nodes_with_client+1 +fi + +for nodenum in `seq $nodes_with_client`; do + options="" + nodename=uml$nodenum + +# if test $nodenum -eq $nodes_with_client; then +# options="--lustre_upcall $upcall" +# fi + + ${LMC} -m $config --add node --node $nodename || exit 10 + ${LMC} -m $config --add net --node $nodename --nid $nodename \ + --nettype tcp $options || exit 11 +done + +# configuring metadata bits +${LMC} -m $config --add lmv --lmv lmv1 || exit 12 + +fonum=1 + +for nodenum in `seq $NODECOUNT`; do + nodename=uml$nodenum + for mdsnum in `seq $MDSCOUNT`; do + options="" + mdsid=mds$mdsnum + + if test $mdsnum -le 2 && test $nodenum -le 2; then + mdsname="$nodename-$mdsid" + mdsdev=$TMP/$nodename-$mdsid + else + options="--failover" + mdsname="failover$fonum" + mdsdev="$TMP/failover$fonum" + let fonum=fonum+1 + fi + + ${LMC} -m $config --format --add mds --node $nodename \ + --mds $mdsname --lmv lmv1 --fstype $FSTYPE --dev $mdsdev \ + --size $MDSSIZE $options || exit 13 + done +done + +# configuring object storage bits +${LMC} -m $config --add lov --lmv lmv1 --lov lov1 --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 || exit 20 +${LMC} -m $config --add ost --ost ost1 --nspath /mnt/ost_ns --node uml2 --lov lov1 --fstype $FSTYPE --dev $OSTDEV --size $OSTSIZE $JARG || exit 30 + +# configuring client +${LMC} -m $config --add mtpt --node uml3 --path $MOUNT --lmv lmv1 --lov lov1 || exit 40 diff --git a/lustre/tests/upcall b/lustre/tests/upcall new file mode 100755 index 0000000..0fa29e1 --- /dev/null +++ b/lustre/tests/upcall @@ -0,0 +1,12 @@ +#!/bin/bash + +active_nid="uml3" +target_name=`echo $2 | sed 's/_.*//'` + +config="$lustre_path/tests/failover.xml" +lustre_path="/r/home/umka/work/cfs/lustre" + +if test "x$1" = "xFAILED_IMPORT"; then + $lustre_path/utils/lconf --recover --select $target_name=$active_nid --tgt_uuid $2 \ + --client_uuid $3 --conn_uuid $4 $config +fi -- 1.8.3.1