From d1f56f0cdab62068c19bd83b3d7db5b44943fb87 Mon Sep 17 00:00:00 2001 From: wangdi Date: Sun, 5 Jun 2005 06:33:04 +0000 Subject: [PATCH] Branch: HEAD some fixes for WB cache 1)reorganization for cobd_setup, add cache_real_exp, master_real_exp for real connection 2)the fix in lmc for wrong lov_index --- lustre/cmobd/cm_obd.c | 3 +- lustre/cobd/cache_obd.c | 95 +++++++++++++++++++++++++++++------------ lustre/include/linux/obd.h | 2 + lustre/tests/cmobd.sh | 104 ++++++++++++++++++++------------------------- lustre/utils/lmc | 12 ++++-- 5 files changed, 124 insertions(+), 92 deletions(-) diff --git a/lustre/cmobd/cm_obd.c b/lustre/cmobd/cm_obd.c index 43fa1e3..3492fe1 100644 --- a/lustre/cmobd/cm_obd.c +++ b/lustre/cmobd/cm_obd.c @@ -154,7 +154,8 @@ static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf) /* connecting master */ memset(&conn, 0, sizeof(conn)); - rc = obd_connect(&conn, cmobd->master_obd, &obd->obd_uuid, NULL, 0); + rc = obd_connect(&conn, cmobd->master_obd, &obd->obd_uuid, + NULL, OBD_OPT_REAL_CLIENT); if (rc) RETURN(rc); cmobd->master_exp = class_conn2export(&conn); diff --git a/lustre/cobd/cache_obd.c b/lustre/cobd/cache_obd.c index 67d69a6..bca2a8c 100644 --- a/lustre/cobd/cache_obd.c +++ b/lustre/cobd/cache_obd.c @@ -96,7 +96,7 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf) /* connecting master */ memset(&conn, 0, sizeof(conn)); - rc = obd_connect(&conn, master_obd, &obd->obd_uuid, NULL, 0); + rc = class_connect(&conn, master_obd, &obd->obd_uuid); if (rc) GOTO(put_names, rc); @@ -105,7 +105,7 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf) /* getting master obd */ cache_obd = class_name2obd(cobd->cache_name); if (!cache_obd) { - obd_disconnect(cobd->master_exp, 0); + class_disconnect(cobd->master_exp, 0); CERROR("can't find cache obd by name %s\n", cobd->cache_name); GOTO(put_names, rc); @@ -113,12 +113,14 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf) /* connecting master */ memset(&conn, 0, sizeof(conn)); - rc = obd_connect(&conn, cache_obd, &obd->obd_uuid, NULL, 0); + rc = class_connect(&conn, cache_obd, &obd->obd_uuid); if (rc) { - obd_disconnect(cobd->master_exp, 0); + class_disconnect(cobd->master_exp, 0); GOTO(put_names, rc); } cobd->cache_exp = class_conn2export(&conn); + /*default set cache on*/ + cobd->cache_on = 1; EXIT; put_names: if (rc) { @@ -138,6 +140,7 @@ put_names: static int cobd_cleanup(struct obd_device *obd, int flags) { struct cache_obd *cobd = &obd->u.cobd; + int rc = 0; ENTRY; if (!list_empty(&obd->obd_exports)) @@ -149,7 +152,18 @@ static int cobd_cleanup(struct obd_device *obd, int flags) if (cobd->master_name) OBD_FREE(cobd->master_name, strlen(cobd->master_name) + 1); - + + rc = class_disconnect(cobd->master_exp, flags); + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + rc = class_disconnect(cobd->cache_exp, flags); + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + RETURN(0); } @@ -157,12 +171,19 @@ static inline struct obd_export * cobd_get_exp(struct obd_device *obd) { struct cache_obd *cobd = &obd->u.cobd; - if (cobd->cache_on) + if (cobd->cache_on) { + CDEBUG(D_TRACE, "get cache exp %p \n", cobd->cache_exp); + if (cobd->cache_real_exp) + return cobd->cache_real_exp; return cobd->cache_exp; + } + CDEBUG(D_TRACE, "get master exp %p \n", cobd->master_exp); + if (cobd->master_real_exp) + return cobd->master_real_exp; return cobd->master_exp; } -static int client_obd_connect(struct obd_device *obd, char *name, +static int client_obd_connect(struct obd_device *obd, struct obd_export *exp, struct lustre_handle *conn, struct obd_connect_data *data, unsigned long flags) @@ -172,20 +193,16 @@ static int client_obd_connect(struct obd_device *obd, char *name, ENTRY; LASSERT(obd); - LASSERT(name); LASSERT(conn); - cli_obd = class_name2obd(name); - if (cli_obd == NULL) { - CERROR("%s: unable to find a client for obd: %s\n", - obd->obd_name, name); + cli_obd = class_exp2obd(exp); + if (cli_obd == NULL) RETURN(-EINVAL); - } + rc = obd_connect(conn, cli_obd, &obd->obd_uuid, data, flags); - if (rc) { - CERROR("error connecting to %s, err %d\n", - name, rc); - } + if (rc) + CERROR("error connecting err %d\n", rc); + RETURN(rc); } @@ -214,12 +231,34 @@ cobd_connect(struct lustre_handle *conn, struct obd_device *obd, struct obd_uuid *cluuid, struct obd_connect_data *data, unsigned long flags) { - struct obd_export *cobd_exp; + struct lustre_handle cache_conn = { 0 }; + struct cache_obd *cobd = &obd->u.cobd; + struct obd_export *exp, *cobd_exp; + int rc = 0; + ENTRY; + + rc = class_connect(conn, obd, cluuid); + if (rc) + RETURN(rc); + exp = class_conn2export(conn); cobd_exp = cobd_get_exp(obd); - return obd_connect(conn, class_exp2obd(cobd_exp), cluuid, - data, flags); + /* connecting cache */ + rc = client_obd_connect(obd, cobd_exp, &cache_conn, + data, flags); + if (rc) + GOTO(err_discon, rc); + + cobd->cache_real_exp = class_conn2export(&cache_conn); + cobd->cache_on = 1; + EXIT; +err_discon: + if (rc) + class_disconnect(exp, 0); + else + class_export_put(exp); + RETURN(rc); } static int @@ -239,8 +278,8 @@ cobd_disconnect(struct obd_export *exp, unsigned long flags) } cobd_exp = cobd_get_exp(obd); - rc = obd_disconnect(cobd_exp, flags); - + rc = client_obd_disconnect(obd, cobd_exp, flags); + class_disconnect(exp, flags); RETURN(rc); @@ -742,12 +781,12 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp, if (!cobd->cache_on) { struct lustre_handle conn = {0}; - rc = client_obd_disconnect(obd, cobd->master_exp, 0); - rc = client_obd_connect(obd, cobd->cache_name, &conn, + rc = client_obd_disconnect(obd, cobd->master_real_exp, 0); + rc = client_obd_connect(obd, cobd->cache_exp, &conn, NULL, 0); if (rc) GOTO(out, rc); - cobd->cache_exp = class_conn2export(&conn); + cobd->cache_real_exp = class_conn2export(&conn); cobd->cache_on = 1; } break; @@ -762,12 +801,12 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp, easize = cache->u.cli.cl_max_mds_easize; cooksize = cache->u.cli.cl_max_mds_cookiesize; - rc = client_obd_disconnect(obd, cobd->cache_exp, 0); - rc = client_obd_connect(obd, cobd->master_name, &conn, + rc = client_obd_disconnect(obd, cobd->cache_real_exp, 0); + rc = client_obd_connect(obd, cobd->master_exp, &conn, NULL, 0); if (rc) GOTO(out, rc); - cobd->master_exp = class_conn2export(&conn); + cobd->master_real_exp = class_conn2export(&conn); master = class_exp2obd(cobd->master_exp); master->u.cli.cl_max_mds_easize = easize; diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index 260b94b..f264325 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -455,6 +455,8 @@ struct echo_client_obd { struct cache_obd { struct obd_export *master_exp; /* local connection to master obd */ struct obd_export *cache_exp; /* local connection to cache obd */ + struct obd_export *cache_real_exp; + struct obd_export *master_real_exp; struct obd_device *master; struct obd_device *cache; char *master_name; diff --git a/lustre/tests/cmobd.sh b/lustre/tests/cmobd.sh index c36f83e..109c588 100644 --- a/lustre/tests/cmobd.sh +++ b/lustre/tests/cmobd.sh @@ -6,10 +6,13 @@ config=${1:-cmobd.xml} LMC=${LMC:-lmc} TMP=${TMP:-/tmp} -CMOBD_MDS1=${CMOBD_MDS1:-"cmobd-mds1"} +COBD_MDS=${COBD_MDS:-"cobd_mds"} +COBD_OST=${COBD_OST:-"cobd_ost"} +CMOBD_MDS=${CMOBD_MDS:-"cmobd-mds"} CMOBD_MDS2=${CMOBD_MDS2:-"cmobd-mds2"} MASTER_LMV=${MASTER_LMV1:-master-lmv1} +CACHE_LMV=${MASTER_LMV1:-cache-lmv1} CACHE_MDS1=${CACHE_MDS1:-"cache-mds1"} CACHE_MDS2=${CACHE_MDS2:-"cache-mds2"} @@ -27,8 +30,11 @@ OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"kml"} MDSSIZE=${MDSSIZE:-100000} MASTER_LOV=${MASTER_LOV:-"master-lov1"} +CACHE_LOV=${CACHE_LOV:-"cache-lov1"} MASTER_OST=${MASTER_OST:-"master-ost1"} +CACHE_OST=${CACHE_OST:-"cache-ost1"} OST_MASTER_DEV=$TMP/ost1-master-localhost +OST_CACHE_DEV=$TMP/ost1-cache-localhost OSTSIZE=${OSTSIZE:-100000} @@ -41,12 +47,10 @@ BACK_FSTYPE=${BACK_FSTYPE:-ldiskfs} NETTYPE=${NETTYPE:-tcp} NIDTYPE=${NIDTYPE:-$NETTYPE} +STRIPE_SIZE=${STRIPE_SIZE:-65536} NODE=${NODE:-"localhost"} -CLIENTS=${CLIENTS:-1} -MODE=${MODE:-lmv} - rm -f $config h2tcp () { @@ -59,64 +63,46 @@ h2tcp () { ${LMC} -m $config --add net --node $NODE --nid `h2$NIDTYPE $NODE` \ --nettype $NETTYPE || exit 1 -${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS1 --fstype $FSTYPE \ ---backfstype $BACK_FSTYPE --dev $MDS1_CACHE_DEV \ ---mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10 - -if test "x$CLIENTS" = "x2"; then - ${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS2 \ - --fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS2_CACHE_DEV \ - --mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10 -fi - -if test "x$MODE" = "xmds"; then - ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \ - --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE --format || exit 10 -else - ${LMC} -m $config --add lmv --lmv $MASTER_LMV || exit 12 - - ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \ - --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \ - --lmv $MASTER_LMV --format || exit 10 - - ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS2 \ - --fstype $BACK_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \ - --lmv $MASTER_LMV --format || exit 10 -fi - -if test "x$MODE" = "xmds"; then - ${LMC} -m $config --add lov --lov $MASTER_LOV --mds $MASTER_MDS1 \ - --stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 -else - ${LMC} -m $config --add lov --lov $MASTER_LOV --lmv $MASTER_LMV \ - --stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 -fi + +#add cache lmv for cache lmv +${LMC} -m $config --add lmv --lmv $CACHE_LMV || exit 12 + +${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS1 \ +--fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS1_CACHE_DEV \ +--mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --lmv $CACHE_LMV --format || exit 10 + +${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS2 \ +--fstype $FSTYPE --backfstype $BACK_FSTYPE --dev $MDS2_CACHE_DEV \ +--mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --lmv $CACHE_LMV --format || exit 10 + +#add master lmv for master lmv +${LMC} -m $config --add lmv --lmv $MASTER_LMV || exit 12 + +${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \ +--fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \ +--lmv $MASTER_LMV --format || exit 10 + +${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS2 \ +--fstype $BACK_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \ +--lmv $MASTER_LMV --format || exit 10 + +${LMC} -m $config --add lov --lov $MASTER_LOV --lmv $CACHE_LMV \ +--stripe_sz $STRIPE_SIZE --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 + +${LMC} -m $config --add lov --lov $CACHE_LOV --lmv $MASTER_LMV \ +--stripe_sz $STRIPE_SIZE --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 ${LMC} -m $config --add ost --ost $MASTER_OST --node $NODE --lov $MASTER_LOV \ --fstype $BACK_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE || exit 21 -if test "x$MODE" = "xmds"; then - ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS1 \ - --master_obd $MASTER_MDS1 --cache_obd $CACHE_MDS1 || exit 23 -else - ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS1 \ - --master_obd $MASTER_LMV --cache_obd $CACHE_MDS1 || exit 23 -fi - -if test "x$CLIENTS" = "x2"; then - if test "x$MODE" = "xmds"; then - ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS2 \ - --master_obd $MASTER_MDS1 --cache_obd $CACHE_MDS2 || exit 23 - else - ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS2 \ - --master_obd $MASTER_LMV --cache_obd $CACHE_MDS2 || exit 23 - fi -fi +${LMC} -m $config --add ost --ost $CACHE_OST --node $NODE --lov $CACHE_LOV \ +--fstype $BACK_FSTYPE --dev $OST_CACHE_DEV --size $OSTSIZE || exit 21 -${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre \ ---mds $CACHE_MDS1 --lov $MASTER_LOV || exit 30 +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS \ +--master_obd $MASTER_LMV --cache_obd $CACHE_LMV || exit 22 + +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST \ +--master_obd $MASTER_LOV --cache_obd $CACHE_LOV || exit 22 -if test "x$CLIENTS" = "x2"; then - ${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre1 \ - --mds $CACHE_MDS2 --lov $MASTER_LOV || exit 30 -fi +${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre \ +--lmv $COBD_MDS --lov $COBD_OST || exit 30 diff --git a/lustre/utils/lmc b/lustre/utils/lmc index 44d6648..35b78be 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -496,9 +496,10 @@ class GenConfig: lov.setAttribute("stripepattern", str(pattern)) return lov - def lov_tgt(self, obd_uuid, index, generation): + def lov_tgt(self, obd_uuid, lov_uuid, index, generation): tgt = self.doc.createElement('lov_tgt') tgt.setAttribute("uuidref", obd_uuid) + tgt.setAttribute("lov_uuid", lov_uuid) tgt.setAttribute("index", index) tgt.setAttribute("generation", generation) tgt.setAttribute("active", '1') @@ -730,6 +731,7 @@ def lov_mod_obd(gen, lustre, lov, tgt, osc_uuid, options): def lov_add_obd(gen, lustre, lov, osc_uuid, options): lov_name = getName(lov) + lov_uuid = getUUID(lov) if options.index: lov_index = get_option_int(options, 'index') for tgt in lustre.getElementsByTagName('lov_tgt'): @@ -745,16 +747,18 @@ def lov_add_obd(gen, lustre, lov, osc_uuid, options): for tgt in lustre.getElementsByTagName('lov_tgt'): uuidref = tgt.getAttribute('uuidref') tmp = int(tgt.getAttribute('index')) + own_lov_uuid = tgt.getAttribute('lov_uuid') if tmp != lov_index: error('malformed xml: LOV targets are not ordered; found index '+str(tmp)+', expected '+str(lov_index)+'.') uuidref = tgt.getAttribute('uuidref') if uuidref == '': lov_mod_obd(gen, lustre, lov, tgt, osc_uuid, options) return - lov_index = lov_index + 1 + if own_lov_uuid == lov_uuid: + lov_index = lov_index + 1 - lov.appendChild(gen.lov_tgt(osc_uuid, str(lov_index), '1')) - addUpdate(gen, lustre, gen.add(getUUID(lov), osc_uuid, str(lov_index), '1')) + lov.appendChild(gen.lov_tgt(osc_uuid, lov_uuid, str(lov_index), '1')) + addUpdate(gen, lustre, gen.add(getUUID(lov), lov_uuid, str(lov_index), '1')) def lov_del_obd(gen, lustre, lov, osc_uuid, options): lov_name = getName(lov) -- 1.8.3.1