From b258207ff5445ae0f1495968c7cb5b9dc7d66047 Mon Sep 17 00:00:00 2001 From: yury Date: Tue, 23 Nov 2004 18:34:12 +0000 Subject: [PATCH] - lots of fixes and cleanups in cobd and cmobd. - fixes and clanups in cmobd.sh and cobd.sh - changes in lconf about module loading in OSD and MDSDEV. - fixes in lconf about mount options forming. - fixes in lmc about client ref managing. - fixes in lconf about cmobd and cobd setup. --- lustre/cmobd/cm_mds_reint.c | 16 ++-- lustre/cmobd/cm_obd.c | 145 ++++++++++++++++++------------- lustre/cmobd/cm_oss_reint.c | 46 ++++++++-- lustre/cmobd/cm_write.c | 9 +- lustre/cobd/cache_obd.c | 188 +++++++++++++++++++--------------------- lustre/include/linux/obd.h | 24 ++--- lustre/lov/lov_obd.c | 10 +-- lustre/obdclass/genops.c | 8 +- lustre/tests/cmobd.sh | 14 +-- lustre/tests/cobd.sh | 85 ++++++++++++++---- lustre/tests/uml_cobd.sh | 101 --------------------- lustre/utils/Lustre/lustredb.py | 16 +++- lustre/utils/lconf | 128 ++++++++++++++------------- lustre/utils/lmc | 44 ++++++++-- 14 files changed, 437 insertions(+), 397 deletions(-) delete mode 100644 lustre/tests/uml_cobd.sh diff --git a/lustre/cmobd/cm_mds_reint.c b/lustre/cmobd/cm_mds_reint.c index 1c76cef..ba544c0 100644 --- a/lustre/cmobd/cm_mds_reint.c +++ b/lustre/cmobd/cm_mds_reint.c @@ -103,7 +103,7 @@ static int cmobd_reint_setattr(struct obd_device *obd, void *record) * #249. Will be fixed later. */ /* converting localstore cookie to remote lustre_id. */ - rc = mds_read_mid(cmobd->cache_obd, &rec->sa_id, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->sa_id, &rec->sa_id, sizeof(rec->sa_id)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -161,7 +161,7 @@ static int cmobd_reint_create(struct obd_device *obd, void *record) memset(&rec->cr_replayid, 0, sizeof(rec->cr_replayid)); /* converting local inode store cookie to remote lustre_id. */ - rc = mds_read_mid(cmobd->cache_obd, &rec->cr_id, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->cr_id, &rec->cr_id, sizeof(rec->cr_id)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -196,7 +196,7 @@ static int cmobd_reint_create(struct obd_device *obd, void *record) * inode EA. */ body = lustre_msg_buf(req->rq_repmsg, 0, sizeof(*body)); - rc = mds_update_mid(cmobd->cache_obd, &lid, + rc = mds_update_mid(cmobd->cache_exp->exp_obd, &lid, &body->id1, sizeof(body->id1)); } @@ -225,7 +225,7 @@ static int cmobd_reint_unlink(struct obd_device *obd, void *record) RETURN(-EINVAL); /* converting local store cookie to remote lustre_id. */ - rc = mds_read_mid(cmobd->cache_obd, &rec->ul_id1, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->ul_id1, &rec->ul_id1, sizeof(rec->ul_id1)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -273,7 +273,7 @@ static int cmobd_reint_link(struct obd_device *obd, void *record) RETURN(-EINVAL); /* converting local store cookie for both ids to remote lustre_id. */ - rc = mds_read_mid(cmobd->cache_obd, &rec->lk_id1, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->lk_id1, &rec->lk_id1, sizeof(rec->lk_id1)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -281,7 +281,7 @@ static int cmobd_reint_link(struct obd_device *obd, void *record) RETURN(rc); } - rc = mds_read_mid(cmobd->cache_obd, &rec->lk_id2, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->lk_id2, &rec->lk_id2, sizeof(rec->lk_id2)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -329,7 +329,7 @@ static int cmobd_reint_rename(struct obd_device *obd, void *record) RETURN(-EINVAL); /* converting local store cookie for both ids to remote lustre_id. */ - rc = mds_read_mid(cmobd->cache_obd, &rec->rn_id1, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->rn_id1, &rec->rn_id1, sizeof(rec->rn_id1)); if (rc) { CERROR("Can't read master MDS store cookie " @@ -337,7 +337,7 @@ static int cmobd_reint_rename(struct obd_device *obd, void *record) RETURN(rc); } - rc = mds_read_mid(cmobd->cache_obd, &rec->rn_id2, + rc = mds_read_mid(cmobd->cache_exp->exp_obd, &rec->rn_id2, &rec->rn_id2, sizeof(rec->rn_id2)); if (rc) { CERROR("Can't read master MDS store cookie " diff --git a/lustre/cmobd/cm_obd.c b/lustre/cmobd/cm_obd.c index 5d6592d..f672def 100644 --- a/lustre/cmobd/cm_obd.c +++ b/lustre/cmobd/cm_obd.c @@ -50,31 +50,6 @@ static int cmobd_detach(struct obd_device *obd) return lprocfs_obd_detach(obd); } -static struct obd_device *cmobd_find_master(struct obd_device *obd, - struct obd_uuid *uuid) -{ - struct obd_device *master_obd; - - CWARN("%s: looking for client obd %s\n", - obd->obd_uuid.uuid, uuid->uuid); - - master_obd = class_find_client_obd(NULL, - OBD_LOV_DEVICENAME, - uuid); - if (master_obd) - return master_obd; - - master_obd = class_find_client_obd(NULL, - OBD_LMV_DEVICENAME, - uuid); - if (master_obd) - return master_obd; - - master_obd = class_find_client_obd(NULL, LUSTRE_MDC_NAME, - uuid); - return master_obd; -} - static inline int cmobd_lmv_obd(struct obd_device *obd) { if (!strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) || @@ -98,19 +73,43 @@ static void cmobd_init_ea_size(struct obd_device *obd) struct cm_obd *cmobd = &obd->u.cm; ENTRY; - /* FIXME-UMKA: here we should also take into account that there is - * possible to have few OSTs. */ + /* + * here we should also take into account that there is possible to have + * few OSTs. --umka + */ easize = lov_mds_md_size(ost_count); cookiesize = ost_count * sizeof(struct llog_cookie); obd_init_ea_size(cmobd->master_exp, easize, cookiesize); - cmobd->master_obd->u.cli.cl_max_mds_easize = easize; - cmobd->master_obd->u.cli.cl_max_mds_cookiesize = cookiesize; + cmobd->master_exp->exp_obd->u.cli.cl_max_mds_easize = easize; + cmobd->master_exp->exp_obd->u.cli.cl_max_mds_cookiesize = cookiesize; EXIT; } +static struct obd_device * +find_master_obd(struct obd_device *obd, struct obd_uuid *uuid) +{ + struct obd_device *master; + + CWARN("%s: looking for client obd %s\n", + obd->obd_uuid.uuid, uuid->uuid); + + master = class_find_client_obd(NULL, OBD_LOV_DEVICENAME, + uuid); + if (master) + return master; + + master = class_find_client_obd(NULL, OBD_LMV_DEVICENAME, + uuid); + if (master) + return master; + + return class_find_client_obd(NULL, LUSTRE_MDC_NAME, + uuid); +} + static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf) { struct obd_uuid master_uuid, cache_uuid; @@ -121,51 +120,57 @@ static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf) int valsize, rc; ENTRY; - if (lcfg->lcfg_inllen1 < 1 || !lcfg->lcfg_inlbuf1) { - CERROR("CMOBD setup requires master uuid\n"); + if (lcfg->lcfg_inllen1 == 0 || lcfg->lcfg_inlbuf1 == NULL) { + CERROR("%s: setup requires master device uuid\n", + obd->obd_name); RETURN(-EINVAL); } - if (lcfg->lcfg_inllen2 < 1 || !lcfg->lcfg_inlbuf2) { - CERROR("CMOBD setup requires cache uuid\n"); + + if (lcfg->lcfg_inllen2 == 0 || lcfg->lcfg_inlbuf2 == NULL) { + CERROR("%s: setup requires cache device uuid\n", + obd->obd_name); RETURN(-EINVAL); } - + obd_str2uuid(&master_uuid, lcfg->lcfg_inlbuf1); obd_str2uuid(&cache_uuid, lcfg->lcfg_inlbuf2); - /* FIXME-WANGDI: saving client obds here is not correct as they may - become invalid due to refcounter exhausting on cleanup. The prefer - way seems to be getting them each time we need them. */ - cmobd->master_obd = cmobd_find_master(obd, &master_uuid); - if (cmobd->master_obd == NULL) { - CERROR("Can't find master obd %s\n", &master_uuid.uuid[0]); + /* getting master obd */ + cmobd->master_obd = find_master_obd(obd, &master_uuid); + if (!cmobd->master_obd) { + CERROR("can't find master obd by uuid %s\n", + master_uuid.uuid); RETURN(-EINVAL); } - + + /* getting cache obd */ cmobd->cache_obd = class_uuid2obd(&cache_uuid); if (cmobd->cache_obd == NULL) { - CERROR("Can't find cache obd %s\n", &cache_uuid.uuid[0]); + CERROR("CMOBD: unable to find obd by uuid: %s\n", + cache_uuid.uuid); RETURN(-EINVAL); } + /* connecting master */ + memset(&conn, 0, sizeof(conn)); rc = obd_connect(&conn, cmobd->master_obd, &obd->obd_uuid, 0); if (rc) RETURN(rc); cmobd->master_exp = class_conn2export(&conn); + /* connecting cache */ memset(&conn, 0, sizeof(conn)); rc = class_connect(&conn, cmobd->cache_obd, &obd->obd_uuid); if (rc) GOTO(put_master, rc); - cmobd->cache_exp = class_conn2export(&conn); - if (cmobd_lov_obd(cmobd->master_obd)) { + if (cmobd_lov_obd(cmobd->master_exp->exp_obd)) { /* for master osc remove the recovery flag. */ rc = obd_set_info(cmobd->master_exp, strlen("unrecovery"), "unrecovery", 0, NULL); if (rc) - GOTO(put_master, rc); + GOTO(put_cache, rc); rc = cmobd_init_write_srv(obd); if (rc) @@ -175,41 +180,47 @@ static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf) cmobd->write_srv = NULL; } - if (cmobd_lmv_obd(cmobd->master_obd)) { - /* making sure, that both obds are ready. This is especially - * important in the case of using LMV as master. */ + if (cmobd_lmv_obd(cmobd->master_exp->exp_obd)) { + /* + * making sure, that both obds are ready. This is especially + * important in the case of using LMV as master. + */ rc = obd_getready(cmobd->master_exp); if (rc) { - CERROR("Can't make %s obd ready.\n", + CERROR("can't get %s obd ready.\n", master_uuid.uuid); GOTO(put_cache, rc); } rc = obd_getready(cmobd->cache_exp); if (rc) { - CERROR("Can't make %s obd ready.\n", + CERROR("can't get %s obd ready.\n", cache_uuid.uuid); GOTO(put_cache, rc); } - /* requesting master obd to have its root inode store cookie to - * be able to save it to local root inode EA. */ + /* + * requesting master obd to have its root inode store cookie to + * be able to save it to local root inode EA. + */ valsize = sizeof(struct lustre_id); rc = obd_get_info(cmobd->master_exp, strlen("rootid"), "rootid", &valsize, &mid); if (rc) { - CERROR("Can't get rootid from master MDS %s, " + CERROR("can't get rootid from master MDS %s, " "err= %d.\n", master_uuid.uuid, rc); GOTO(put_cache, rc); } - /* getting rootid from cache MDS. It is needed to update local - * (cache) root inode by rootid value from master obd. */ + /* + * getting rootid from cache MDS. It is needed to update local + * (cache) root inode by rootid value from master obd. + */ rc = obd_get_info(cmobd->cache_exp, strlen("rootid"), "rootid", &valsize, &lid); if (rc) { - CERROR("Can't get rootid from local MDS %s, " + CERROR("can't get rootid from local MDS %s, " "err= %d.\n", cache_uuid.uuid, rc); GOTO(put_cache, rc); } @@ -218,10 +229,10 @@ static int cmobd_setup(struct obd_device *obd, obd_count len, void *buf) CWARN("storing "DLID4" to local inode "DLID4".\n", OLID4(&mid), OLID4(&lid)); - rc = mds_update_mid(cmobd->cache_obd, &lid, + rc = mds_update_mid(cmobd->cache_exp->exp_obd, &lid, &mid, sizeof(mid)); if (rc) { - CERROR("Can't update local root inode by ID " + CERROR("can't update local root inode by ID " "from master MDS %s, err = %d.\n", master_uuid.uuid, rc); GOTO(put_cache, rc); @@ -233,19 +244,29 @@ put_cache: class_disconnect(cmobd->cache_exp, 0); put_master: obd_disconnect(cmobd->master_exp, 0); - RETURN(rc); + return rc; } static int cmobd_cleanup(struct obd_device *obd, int flags) { struct cm_obd *cmobd = &obd->u.cm; + int rc; ENTRY; if (cmobd->write_srv) cmobd_cleanup_write_srv(obd); - class_disconnect(cmobd->cache_exp, 0); - obd_disconnect(cmobd->master_exp, 0); + rc = obd_disconnect(cmobd->master_exp, flags); + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + + rc = class_disconnect(cmobd->cache_exp, flags); + if (rc) { + CERROR("error disconnecting cache, err %d\n", + rc); + } RETURN(0); } @@ -304,7 +325,7 @@ static int __init cmobd_init(void) RETURN(0); } -static void /*__exit*/ cmobd_exit(void) +static void __exit cmobd_exit(void) { class_unregister_type(LUSTRE_CMOBD_NAME); if (kmem_cache_destroy(cmobd_extent_slab) != 0) diff --git a/lustre/cmobd/cm_oss_reint.c b/lustre/cmobd/cm_oss_reint.c index 88f9e6d..1859063 100644 --- a/lustre/cmobd/cm_oss_reint.c +++ b/lustre/cmobd/cm_oss_reint.c @@ -77,14 +77,21 @@ void cmobd_free_lsm(struct lov_stripe_md **lsmp) /* reintegration functions */ static int cmobd_setattr_reint(struct obd_device *obd, void *rec) { + struct obdo *oa = (struct obdo*)rec; struct cm_obd *cmobd = &obd->u.cm; - struct lov_obd *lov = &cmobd->master_obd->u.lov; struct obd_export *exp = cmobd->master_exp; struct lov_stripe_md *lsm; - struct obdo *oa = (struct obdo*)rec; + struct lov_obd *lov; int rc; ENTRY; + /* + * nevertheless ost is not used anymore and lov should be always present + * as a object storage export, using ost is still possible (just + * deprecated) and we should make sure here, that this is really + * lov. --umka. + */ + lov = &cmobd->master_exp->exp_obd->u.lov; rc = cmobd_dummy_lsm(&lsm, lov->desc.ld_tgt_count, oa, (__u32)lov->desc.ld_default_stripe_size); if (rc) @@ -99,15 +106,22 @@ out: static int cmobd_create_reint(struct obd_device *obd, void *rec) { + struct obdo *oa = (struct obdo *)rec; struct cm_obd *cmobd = &obd->u.cm; - struct lov_obd *lov = &cmobd->master_obd->u.lov; struct obd_export *exp = cmobd->master_exp; struct lov_stripe_md *lsm; struct obd_trans_info oti = { 0 }; - struct obdo *oa = (struct obdo *)rec; + struct lov_obd *lov; int rc; ENTRY; + /* + * nevertheless ost is not used anymore and lov should be always present + * as a object storage export, using ost is still possible (just + * deprecated) and we should make sure here, that this is really + * lov. --umka. + */ + lov = &cmobd->master_exp->exp_obd->u.lov; rc = cmobd_dummy_lsm(&lsm, lov->desc.ld_tgt_count, oa, (__u32)lov->desc.ld_default_stripe_size); if (rc) @@ -203,14 +217,14 @@ static int cmobd_write_extents(struct obd_device *obd, struct obdo *oa, struct ldlm_extent *extent) { struct cm_obd *cmobd = &obd->u.cm; - struct obd_device *cache = cmobd->cache_obd; - struct lov_obd *lov = &cmobd->master_obd->u.lov; - struct ldlm_res_id res_id; - ldlm_policy_data_t policy; + struct obd_device *cache = cmobd->cache_exp->exp_obd; struct lustre_handle lockh_src = { 0 }; struct lustre_handle lockh_dst = { 0 }; + struct ldlm_res_id res_id; + ldlm_policy_data_t policy; struct lov_stripe_md *lsm; int flags = 0, err, rc = 0; + struct lov_obd *lov; ENTRY; /* XXX for debug write replay without smfs and kml */ @@ -227,7 +241,21 @@ static int cmobd_write_extents(struct obd_device *obd, struct obdo *oa, if (rc != ELDLM_OK) RETURN(rc); + /* + * nevertheless ost is not used anymore and lov should be always present + * as a object storage export, using ost is still possible (just + * deprecated) and we should make sure here, that this is really + * lov. --umka. + */ + lov = &cmobd->master_exp->exp_obd->u.lov; + /* construct the pseudo lsm */ + + /* + * it is not good to access lov fields like @desc directly. This is + * layering violation. It should be accessed via some interface method, + * like llite does. --umka + */ rc = cmobd_dummy_lsm(&lsm, lov->desc.ld_tgt_count, oa, (__u32)lov->desc.ld_default_stripe_size); if (rc) @@ -260,7 +288,7 @@ out_lock: static int cmobd_write_reint(struct obd_device *obd, void *rec) { struct cm_obd *cmobd = &obd->u.cm; - struct obd_device *cache = cmobd->cache_obd; + struct obd_device *cache = cmobd->cache_exp->exp_obd; struct obdo *oa = (struct obdo *)rec; struct ldlm_extent *extent = NULL; unsigned long csb, ino; diff --git a/lustre/cmobd/cm_write.c b/lustre/cmobd/cm_write.c index e3e12b3..201b199 100644 --- a/lustre/cmobd/cm_write.c +++ b/lustre/cmobd/cm_write.c @@ -651,19 +651,26 @@ int cmobd_replay_write(struct obd_device *obd, struct obdo *oa, struct ldlm_extent *ext) { struct cm_obd *cmobd = &obd->u.cm; - struct lov_obd *lov = &cmobd->master_obd->u.lov; struct lov_stripe_md *lsm = NULL; struct cmobd_extent_set set; struct cmobd_extent_info *ex; struct l_wait_info lwi = { 0 }; struct list_head *pos, *n; struct cmobd_async_page *cmap; + struct lov_obd *lov; unsigned long flags; obd_count i, buf_count; obd_off start; int rc = 0; ENTRY; + /* + * nevertheless ost is not used anymore and lov should be always present + * as a object storage export, using ost is still possible (just + * deprecated) and we should make sure here, that this is really + * lov. --umka. + */ + lov = &cmobd->master_exp->exp_obd->u.lov; rc = cmobd_dummy_lsm(&lsm, lov->desc.ld_tgt_count, oa, (__u32)lov->desc.ld_default_stripe_size); if (rc) diff --git a/lustre/cobd/cache_obd.c b/lustre/cobd/cache_obd.c index 15e2b88..204ee8a 100644 --- a/lustre/cobd/cache_obd.c +++ b/lustre/cobd/cache_obd.c @@ -31,6 +31,7 @@ #include #include #include +#include static int cobd_attach(struct obd_device *obd, obd_count len, void *buf) { @@ -47,15 +48,15 @@ static int cobd_detach(struct obd_device *obd) static int connect_to_obd(char *name, struct lustre_handle *conn) { - struct obd_uuid obd_uuid; + struct obd_uuid obd_uuid; struct obd_device *obd; - int rc = 0; + int rc = 0; ENTRY; obd = class_name2obd(name); if (obd == NULL) { - CERROR("%s: unable to find a client for obd: %s\n", - obd->obd_name, name); + CERROR("COBD: unable to find a client for obd: %s\n", + name); RETURN(-EINVAL); } rc = obd_connect(conn, obd, &obd_uuid, 0); @@ -66,12 +67,8 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf) { struct lustre_cfg *lcfg = (struct lustre_cfg *)buf; struct cache_obd *cobd = &obd->u.cobd; -#if 0 - struct lustre_handle master_conn = {0,}; -#endif - struct lustre_handle cache_conn = {0,}; + struct lustre_handle conn = { 0 }; struct obd_device *master; - struct obd_device *cache; int rc; ENTRY; @@ -81,65 +78,41 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf) RETURN(-EINVAL); } - master = class_name2obd(lcfg->lcfg_inlbuf1); - if (master == NULL) { - CERROR("%s: unable to find a client for master: %s\n", - obd->obd_name, lcfg->lcfg_inlbuf1); - RETURN(-EINVAL); - } - if (lcfg->lcfg_inllen2 == 0 || lcfg->lcfg_inlbuf2 == NULL) { - CERROR("%s: setup requires cache device name\n", obd->obd_name); + CERROR("%s: setup requires cache device name\n", + obd->obd_name); RETURN(-EINVAL); } - cache = class_name2obd(lcfg->lcfg_inlbuf2); - if (cache == NULL) { - CERROR("%s: unable to find a client for cache: %s\n", - obd->obd_name, lcfg->lcfg_inlbuf2); + master = class_name2obd(lcfg->lcfg_inlbuf1); + if (!master) { + CERROR("%s: unable to find master: %s\n", + obd->obd_name, lcfg->lcfg_inlbuf1); RETURN(-EINVAL); } + cobd->cache_on = 1; + sema_init(&cobd->sem, 1); + OBD_ALLOC(cobd->master_name, strlen(lcfg->lcfg_inlbuf1) + 1); if (!cobd->master_name) - GOTO(exit, rc = -ENOMEM); + GOTO(put_names, rc = -ENOMEM); memcpy(cobd->master_name, lcfg->lcfg_inlbuf1, strlen(lcfg->lcfg_inlbuf1)); OBD_ALLOC(cobd->cache_name, strlen(lcfg->lcfg_inlbuf2) + 1); if (!cobd->cache_name) - GOTO(exit, rc = -ENOMEM); + GOTO(put_names, rc = -ENOMEM); memcpy(cobd->cache_name, lcfg->lcfg_inlbuf2, strlen(lcfg->lcfg_inlbuf2)); -#if 0 - /* don't bother checking attached/setup; obd_connect() should, and it - * can change underneath us */ - rc = connect_to_obd(cobd->master_name, &master_conn); - if (rc != 0) - GOTO(exit, rc); - cobd->master_exp = class_conn2export(&master_conn); -#endif - rc = connect_to_obd(cobd->cache_name, &cache_conn); - if (rc != 0) { - obd_disconnect(cobd->cache_exp, 0); - GOTO(exit, rc); - } - cobd->cache_exp = class_conn2export(&cache_conn); - cobd->cache_on = 1; + rc = connect_to_obd(cobd->cache_name, &conn); + if (rc) + GOTO(put_names, rc); + cobd->cache_exp = class_conn2export(&conn); - if (!strcmp(master->obd_type->typ_name, LUSTRE_MDC_NAME)) { - int mds_type; - - mds_type = MDS_MASTER_OBD; - obd_set_info(cobd->master_exp, strlen("mds_type"), - "mds_type", sizeof(mds_type), &mds_type); - - mds_type = MDS_CACHE_OBD; - obd_set_info(cobd->cache_exp, strlen("mds_type"), - "mds_type", sizeof(mds_type), &mds_type); - } -exit: + RETURN(0); +put_names: if (rc) { if (cobd->cache_name) OBD_FREE(cobd->cache_name, @@ -148,48 +121,52 @@ exit: OBD_FREE(cobd->master_name, strlen(cobd->master_name) + 1); } - RETURN(rc); + return rc; } static int cobd_cleanup(struct obd_device *obd, int flags) { struct cache_obd *cobd = &obd->u.cobd; - int rc; + int rc; + ENTRY; + + if (cobd->cache_on) { + rc = obd_disconnect(cobd->cache_exp, flags); + if (rc) { + CERROR("error disconnecting cache, err %d\n", + rc); + } + } else { + rc = obd_disconnect(cobd->master_exp, flags); + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + } - if (!list_empty(&obd->obd_exports)) - return (-EBUSY); - if (cobd->cache_name) OBD_FREE(cobd->cache_name, strlen(cobd->cache_name) + 1); if (cobd->master_name) OBD_FREE(cobd->master_name, strlen(cobd->master_name) + 1); - if (cobd->cache_on) { - rc = obd_disconnect(cobd->cache_exp, flags); - if (rc != 0) - CERROR("error %d disconnecting cache\n", rc); - } - rc = obd_disconnect(cobd->master_exp, flags); - if (rc != 0) - CERROR("error %d disconnecting master\n", rc); - return (rc); + RETURN(rc); } struct obd_export *cobd_get_exp(struct obd_device *obd) { - struct cache_obd *cobd = &obd->u.cobd; + struct cache_obd *cobd = &obd->u.cobd; if (cobd->cache_on) return cobd->cache_exp; - else - return cobd->master_exp; + + return cobd->master_exp; } static int cobd_connect(struct lustre_handle *conn, struct obd_device *obd, - struct obd_uuid *cluuid, unsigned long connect_flags) + struct obd_uuid *cluuid, unsigned long flags) { return class_connect(conn, obd, cluuid); } @@ -679,53 +656,60 @@ static int cobd_flush(struct obd_device *obd) return 0; } -static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp, int len, - void *karg, void *uarg) +static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp, + int len, void *karg, void *uarg) { struct obd_device *obd = class_exp2obd(exp); struct cache_obd *cobd = &obd->u.cobd; - struct obd_device *master_dev = NULL; struct obd_export *cobd_exp; int rc = 0; - + ENTRY; + + down(&cobd->sem); + switch (cmd) { case OBD_IOC_COBD_CON: if (!cobd->cache_on) { - struct lustre_handle cache_conn = {0,}; + struct lustre_handle conn = {0,}; rc = obd_disconnect(cobd->master_exp, 0); - if (rc != 0) - CERROR("error %d disconnecting master\n", rc); - rc = connect_to_obd(cobd->cache_name, &cache_conn); - if (rc != 0) - RETURN(rc); - cobd->cache_exp = class_conn2export(&cache_conn); - + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + rc = connect_to_obd(cobd->cache_name, &conn); + if (rc) + GOTO(out, rc); + cobd->cache_exp = class_conn2export(&conn); cobd->cache_on = 1; } break; case OBD_IOC_COBD_COFF: if (cobd->cache_on) { - struct lustre_handle master_conn = {0,}; - struct obd_device *cache_dev = NULL; - int m_easize, m_cooksize; + struct lustre_handle conn = {0,}; + struct obd_device *master = NULL; + struct obd_device *cache = NULL; + int easize, cooksize; - cache_dev = class_exp2obd(cobd->cache_exp); - m_easize = cache_dev->u.cli.cl_max_mds_easize; - m_cooksize = cache_dev->u.cli.cl_max_mds_cookiesize; - rc = obd_disconnect(cobd->cache_exp, 0); - if (rc != 0) - CERROR("error %d disconnecting master\n", rc); + cache = class_exp2obd(cobd->cache_exp); + easize = cache->u.cli.cl_max_mds_easize; + cooksize = cache->u.cli.cl_max_mds_cookiesize; - /* FIXME-WANGDI: should we read from master_dev? */ - - rc = connect_to_obd(cobd->master_name, &master_conn); - if (rc != 0) - RETURN(rc); - cobd->master_exp = class_conn2export(&master_conn); - master_dev = class_exp2obd(cobd->master_exp); - master_dev->u.cli.cl_max_mds_easize = m_easize; - master_dev->u.cli.cl_max_mds_cookiesize = m_cooksize; + rc = obd_disconnect(cobd->cache_exp, 0); + if (rc) { + CERROR("error disconnecting master, err %d\n", + rc); + } + + /* should we read from master_dev? */ + rc = connect_to_obd(cobd->master_name, &conn); + if (rc) + GOTO(out, rc); + cobd->master_exp = class_conn2export(&conn); + master = class_exp2obd(cobd->master_exp); + + master->u.cli.cl_max_mds_easize = easize; + master->u.cli.cl_max_mds_cookiesize = cooksize; cobd->cache_on = 0; } break; @@ -733,13 +717,19 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp, int len, if (cobd->cache_on) { cobd->cache_on = 0; cobd_flush(obd); + } else { + CERROR("%s: cache is turned off\n", + obd->obd_name); } break; default: cobd_exp = cobd_get_exp(obd); rc = obd_iocontrol(cmd, cobd_exp, len, karg, uarg); } - + + EXIT; +out: + up(&cobd->sem); return rc; } diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index 8177523..283b1db 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -424,14 +424,14 @@ struct recovd_obd { }; struct ost_obd { - spinlock_t ost_lock; + spinlock_t ost_lock; struct ptlrpc_service *ost_service; struct ptlrpc_service *ost_create_service; struct obd_service_time ost_stimes[6]; }; struct echo_client_obd { - struct obd_export *ec_exp; /* the local connection to osc/lov */ + struct obd_export *ec_exp; /* the local connection to osc/lov */ spinlock_t ec_lock; struct list_head ec_objects; int ec_nstripes; @@ -445,8 +445,18 @@ struct cache_obd { char *cache_name; int refcount; int cache_on; + struct semaphore sem; }; +struct cm_obd { + struct obd_export *cache_exp; /* local connection to cache obd */ + struct obd_export *master_exp; + struct obd_device *cache_obd; + struct obd_device *master_obd; + int master_group; + struct cmobd_write_service *write_srv; +}; + struct lov_tgt_desc { struct obd_uuid uuid; __u32 ltd_gen; @@ -498,16 +508,6 @@ struct niobuf_local { int rc; }; -struct cm_obd { - struct obd_device *master_obd; /* master lov */ - struct obd_export *master_exp; - struct obd_device *cache_obd; /* cache obdfilter */ - struct obd_export *cache_exp; - int master_group; /* master group*/ - struct cmobd_write_service *write_srv; -}; - - /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */ #define N_LOCAL_TEMP_PAGE 0x10000000 diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index 2c91efc..ed4612b 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -487,17 +487,17 @@ int lov_detach(struct obd_device *dev) static int lov_setup(struct obd_device *obd, obd_count len, void *buf) { + struct lov_obd *lov = &obd->u.lov; struct lustre_cfg *lcfg = buf; struct lov_desc *desc; - struct lov_obd *lov = &obd->u.lov; int count; ENTRY; - + if (lcfg->lcfg_inllen1 < 1) { CERROR("LOV setup requires a descriptor\n"); RETURN(-EINVAL); } - + desc = (struct lov_desc *)lcfg->lcfg_inlbuf1; if (sizeof(*desc) > lcfg->lcfg_inllen1) { CERROR("descriptor size wrong: %d > %d\n", @@ -528,13 +528,13 @@ static int lov_setup(struct obd_device *obd, obd_count len, void *buf) lov->bufsize); RETURN(-EINVAL); } - + desc->ld_tgt_count = 0; desc->ld_active_tgt_count = 0; lov->desc = *desc; spin_lock_init(&lov->lov_lock); sema_init(&lov->lov_llog_sem, 1); - + RETURN(0); } diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 4c3d3f8..26c96e6 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -271,9 +271,9 @@ struct obd_device *class_uuid2obd(struct obd_uuid *uuid) specified, then only the client with that uuid is returned, otherwise any client connected to the tgt is returned. If tgt_uuid is NULL, the lov with grp_uuid is returned. */ -struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid, - char * typ_name, - struct obd_uuid *grp_uuid) +struct obd_device *class_find_client_obd(struct obd_uuid *tgt_uuid, + char *typ_name, + struct obd_uuid *grp_uuid) { int i; @@ -308,7 +308,7 @@ struct obd_device * class_find_client_obd(struct obd_uuid *tgt_uuid, searching at *next, and if a device is found, the next index to look it is saved in *next. If next is NULL, then the first matching device will always be returned. */ -struct obd_device * class_devices_in_group(struct obd_uuid *grp_uuid, int *next) +struct obd_device *class_devices_in_group(struct obd_uuid *grp_uuid, int *next) { int i; diff --git a/lustre/tests/cmobd.sh b/lustre/tests/cmobd.sh index 35d25f1..123f7ef 100644 --- a/lustre/tests/cmobd.sh +++ b/lustre/tests/cmobd.sh @@ -37,7 +37,7 @@ OSDTYPE=${OSDTYPE:-obdfilter} OSTFAILOVER=${OSTFAILOVER:-} FSTYPE=${FSTYPE:-smfs} -BACKUP_FSTYPE=${BACKUP_FSTYPE:-ext3} +BACK_FSTYPE=${BACK_FSTYPE:-ext3} NETTYPE=${NETTYPE:-tcp} NIDTYPE=${NIDTYPE:-$NETTYPE} @@ -60,27 +60,27 @@ ${LMC} -m $config --add net --node $NODE --nid `h2$NIDTYPE $NODE` \ --nettype $NETTYPE || exit 1 ${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS1 --fstype $FSTYPE \ ---backfstype $BACKUP_FSTYPE --backdev $MDS1_CACHE_DEV --dev $FSTYPE \ +--backfstype $BACK_FSTYPE --backdev $MDS1_CACHE_DEV --dev $FSTYPE \ --mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10 if test "x$CLIENTS" = "x2"; then ${LMC} -m $config --add mds --node $NODE --mds $CACHE_MDS2 \ - --fstype $FSTYPE --backfstype $BACKUP_FSTYPE --backdev $MDS2_CACHE_DEV \ + --fstype $FSTYPE --backfstype $BACK_FSTYPE --backdev $MDS2_CACHE_DEV \ --dev $FSTYPE --mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE --format || exit 10 fi if test "x$MODE" = "xmds"; then ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \ - --fstype $BACKUP_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE --format || exit 10 + --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE --format || exit 10 else ${LMC} -m $config --add lmv --lmv $MASTER_LMV || exit 12 ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS1 \ - --fstype $BACKUP_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \ + --fstype $BACK_FSTYPE --dev $MDS1_MASTER_DEV --size $MDSSIZE \ --lmv $MASTER_LMV --format || exit 10 ${LMC} -m $config --add mds --node $NODE --mds $MASTER_MDS2 \ - --fstype $BACKUP_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \ + --fstype $BACK_FSTYPE --dev $MDS2_MASTER_DEV --size $MDSSIZE \ --lmv $MASTER_LMV --format || exit 10 fi @@ -93,7 +93,7 @@ else fi ${LMC} -m $config --add ost --ost $MASTER_OST --node $NODE --lov $MASTER_LOV \ ---fstype $BACKUP_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE || exit 21 +--fstype $BACK_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE || exit 21 if test "x$MODE" = "xmds"; then ${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS1 \ diff --git a/lustre/tests/cobd.sh b/lustre/tests/cobd.sh index 728407d..9e9e1cd 100755 --- a/lustre/tests/cobd.sh +++ b/lustre/tests/cobd.sh @@ -1,32 +1,79 @@ #!/bin/bash +export PATH=`dirname $0`/../utils:$PATH -config=${1:-$(basename $0 .sh)}.xml - -LMC=${LMC:-../utils/lmc -m $config} +config=${1:-cobd.xml} +LMC=${LMC:-lmc} TMP=${TMP:-/tmp} -MDSDEV=${MDSDEV:-$TMP/mds1-`hostname`} -MDSSIZE=50000 -FSTYPE=${FSTYPE:-ext3} +COBD_MDS=${COBD_MDS:-"cobd1"} +COBD_OST=${COBD_OST:-"cobd2"} + +CACHE_MDS=${CACHE_MDS:-"mds1"} +MASTER_MDS=${MASTER_MDS:-"mds2"} +MDS_CACHE_DEV=$TMP/mds1-`hostname` +MDS_MASTER_DEV=$TMP/mds2-`hostname` +MDSSIZE=${MDSSIZE:-100000} + +CACHE_LOV=${CACHE_LOV:-"lov1"} +MASTER_LOV=${MASTER_LOV:-"lov2"} + +CACHE_OST=${CACHE_OST:-"ost1"} +MASTER_OST=${MASTER_OST:-"ost2"} +OST_CACHE_DEV=$TMP/ost1-`hostname` +OST_MASTER_DEV=$TMP/ost2-`hostname` +OSTSIZE=${OSTSIZE:-100000} + +STRIPECNT=${STRIPECNT:-1} +OSDTYPE=${OSDTYPE:-obdfilter} +OSTFAILOVER=${OSTFAILOVER:-} + +FSTYPE=${FSTYPE:-smfs} +BACK_FSTYPE=${BACK_FSTYPE:-ext3} + +NETTYPE=${NETTYPE:-tcp} +NIDTYPE=${NIDTYPE:-$NETTYPE} -OSTDEV=${OSTDEV:-$TMP/ost1-`hostname`} -OSTSIZE=200000 +MDSNODE=${MDSNODE:-`hostname`} +OSTNODE=${OSTNODE:-`hostname`} +CLIENT=${CLIENT:-`hostname`} +NODE=${NODE:-`hostname`} + +MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"kml"} +OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"kml"} rm -f $config + +h2tcp () { + case $1 in + client) echo '\*' ;; + *) echo $1 ;; + esac +} + # create nodes -${LMC} --add node --node localhost || exit 10 -${LMC} --add net --node localhost --nid localhost --nettype tcp || exit 11 +${LMC} -m $config --add net --node $NODE --nid `h2$NIDTYPE $NODE` --nettype $NETTYPE || exit 1 + +${LMC} -m $config --format --add mds --node $MDSNODE --mds $CACHE_MDS --fstype $FSTYPE \ +--backfstype $BACK_FSTYPE --dev $FSTYPE --backdev $MDS_CACHE_DEV --mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE ||exit 10 + +${LMC} -m $config --format --add mds --node $MDSNODE --mds $MASTER_MDS --fstype $BACK_FSTYPE \ +--dev $MDS_MASTER_DEV --size $MDSSIZE || exit 10 + +${LMC} -m $config --add lov --lov $CACHE_LOV --mds $CACHE_MDS \ +--stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 + +${LMC} -m $config --add lov --lov $MASTER_LOV --mds $MASTER_MDS \ +--stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 -# configure mds server -${LMC} --add mds --node localhost --mds mds1 --fstype $FSTYPE --dev $MDSDEV --size $MDSSIZE || exit 20 +${LMC} -m $config --add ost --node $NODE --lov $CACHE_LOV \ +--fstype $FSTYPE --dev $FSTYPE --backfstype $BACK_FSTYPE \ +--backdev $OST_CACHE_DEV --mountfsoptions $OST_MOUNT_OPTS --size $OSTSIZE || exit 21 -# configure ost -${LMC} --add ost --node localhost --obd obd1 --fstype $FSTYPE --obdtype obdecho || exit 30 -# configure ost -${LMC} --add ost --node localhost --obd obd2 --fstype $FSTYPE --obdtype obdecho || exit 30 +${LMC} -m $config --add ost --node $NODE --lov $MASTER_LOV \ +--fstype $BACK_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE || exit 21 -${LMC} --add cobd --node localhost --master_obd obd1 --cache_obd obd2 +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST --master_obd $MASTER_LOV --cache_obd $CACHE_LOV || exit 22 +${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS --master_obd $MASTER_MDS --cache_obd $CACHE_MDS || exit 22 -# create client config -# ${LMC} -m $config --add mtpt --node localhost --path /mnt/lustre --mds mds1 --obd obd1 || exit 40 +${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre --mds $COBD_MDS --lov $COBD_OST || exit 30 diff --git a/lustre/tests/uml_cobd.sh b/lustre/tests/uml_cobd.sh deleted file mode 100644 index 966ac1e..0000000 --- a/lustre/tests/uml_cobd.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -export PATH=`dirname $0`/../utils:$PATH - -config=${1:-uml_cobd.xml} -LMC=${LMC:-lmc} -TMP=${TMP:-/tmp} - -COBD_MDS=${COBD_MDS:-"cobd1"} -COBD_OST=${COBD_OST:-"cobd2"} - -CMOBD_MDS=${CMOBD_MDS:-"cmobd1"} -CMOBD_OST=${CMOBD_OST:-"cmobd2"} - -CACHE_MDS=${CACHE_MDS:-"mds1"} -MASTER_MDS=${MASTER_MDS:-"mds2"} -MDS_CACHE_DEV=$TMP/mds1-`hostname` -MDS_MASTER_DEV=$TMP/mds2-`hostname` -MDS_DEV=$TMP/mds -MDSSIZE=${MDSSIZE:-100000} - -CACHE_LOV=${CACHE_LOV:-"lov1"} -MASTER_LOV=${MASTER_LOV:-"lov2"} - -CACHE_OST=${CACHE_OST:-"ost1"} -MASTER_OST=${MASTER_OST:-"ost2"} -OST_CACHE_DEV=$TMP/ost1-`hostname` -OST_MASTER_DEV=$TMP/ost2-`hostname` -OST_DEV=$TMP/ost -OSTSIZE=${OSTSIZE:-100000} - - -STRIPECNT=${STRIPECNT:-1} -OSDTYPE=${OSDTYPE:-obdfilter} -OSTFAILOVER=${OSTFAILOVER:-} - -FSTYPE=${FSTYPE:-smfs} -BACKUP_FSTYPE=${BACKUP_FSTYPE:-ext3} - -NETTYPE=${NETTYPE:-tcp} -NIDTYPE=${NIDTYPE:-$NETTYPE} - -MDSNODE=${MDSNODE:-`hostname`} -OSTNODE=${OSTNODE:-`hostname`} -CLIENT=${CLIENT:-`hostname`} -NODE=${NODE:-`hostname`} - -MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"kml"} -OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"kml"} - - -rm -f $config - -h2tcp () { - case $1 in - client) echo '\*' ;; - *) echo $1 ;; - esac -} - - -# create nodes -${LMC} -m $config --add net --node $NODE --nid `h2$NIDTYPE $NODE` --nettype $NETTYPE || exit 1 - -# configure mds server -echo "adding cache MDS on: $MDSNODE" -${LMC} -m $config --format --add mds --node $MDSNODE --mds $CACHE_MDS --fstype $FSTYPE \ ---backfstype $BACKUP_FSTYPE --dev $MDS_DEV --backdev $MDS_CACHE_DEV \ ---mountfsoptions $MDS_MOUNT_OPTS --size $MDSSIZE ||exit 10 - -echo "adding master MDS on: $MDSNODE" -${LMC} -m $config --format --add mds --node $MDSNODE --mds $MASTER_MDS --fstype $BACKUP_FSTYPE \ ---dev $MDS_MASTER_DEV --size $MDSSIZE ||exit 10 - -echo "add cache lov on: $MDSNODE" -${LMC} -m $config --add lov --lov $CACHE_LOV --mds $CACHE_MDS \ ---stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 - -echo "add master lov on: $MDSNODE" -${LMC} -m $config --add lov --lov $MASTER_LOV --mds $MASTER_MDS \ ---stripe_sz 65536 --stripe_cnt $STRIPECNT --stripe_pattern 0 || exit 20 - -echo "add cache ost on $OSTNODE" -${LMC} -m $config --add ost --node $NODE --lov $CACHE_LOV \ ---fstype $FSTYPE --backfstype $BACKUP_FSTYPE --dev $OST_DEV \ ---backdev $OST_CACHE_DEV --mountfsoptions $OST_MOUNT_OPTS --size $OSTSIZE || exit 21 - -echo "add master ost on $OSTNODE" -${LMC} -m $config --add ost --node $NODE --lov $MASTER_LOV \ ---fstype $BACKUP_FSTYPE --dev $OST_MASTER_DEV --size $OSTSIZE || exit 21 - -echo "add mds lov: $COBD_MDS $COBD_OST" -${LMC} -m $config --add cobd --node $NODE --cobd $COBD_OST --master_obd $MASTER_LOV --cache_obd $CACHE_LOV || exit 22 -${LMC} -m $config --add cobd --node $NODE --cobd $COBD_MDS --master_obd $MASTER_MDS --cache_obd $CACHE_MDS || exit 22 -# create client config(s) - -echo "add cmobd: $CMOBD_MDS $CMOBD_OST" -${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_OST --master_obd $MASTER_LOV --cache_obd $CACHE_LOV || exit 23 -${LMC} -m $config --add cmobd --node $NODE --cmobd $CMOBD_MDS --master_obd $MASTER_MDS --cache_obd $CACHE_MDS || exit 23 - -${LMC} -m $config --add mtpt --node $NODE --path /mnt/lustre --mds $COBD_MDS --lov $COBD_OST || exit 30 diff --git a/lustre/utils/Lustre/lustredb.py b/lustre/utils/Lustre/lustredb.py index 12656b5..3d3c4ae 100644 --- a/lustre/utils/Lustre/lustredb.py +++ b/lustre/utils/Lustre/lustredb.py @@ -65,10 +65,15 @@ class LustreDB: return uuids def get_lov_tgts(self, tag): - """ Returns list of lovtgts. """ + """ Returns list of lov tgts. """ tgts = self._get_lov_tgts(tag) return tgts + def get_lmv_tgts(self, tag): + """ Returns list of lmv tgts. """ + tgts = self._get_lmv_tgts(tag) + return tgts + def nid2server(self, nid, net_type, cluster_id): netlist = self.lookup_class('network') for net_db in netlist: @@ -205,6 +210,15 @@ class LustreDB_XML(LustreDB): generation = tgt.getAttribute('generation') active = int(tgt.getAttribute('active')) tgts.append((uuidref, index, generation, active)) + return tgts + + def _get_lmv_tgts(self, tag): + """ Get all the refs of type TAG. Returns list of lmv_tgts. """ + tgts = [] + tgtlist = self.dom_node.getElementsByTagName(tag) + for tgt in tgtlist: + uuidref = tgt.getAttribute('uuidref') + tgts.append((uuidref)) return tgts def xmllookup_by_uuid(self, dom_node, uuid): diff --git a/lustre/utils/lconf b/lustre/utils/lconf index b002276..22bcd23 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -1629,7 +1629,11 @@ class LMV(Module): Module.__init__(self, 'LMV', db) if name_override != None: self.name = "lmv_%s" % name_override - self.devlist = self.db.get_refs('mds') + + self.devlist = self.db.get_lmv_tgts('lmv_tgt') + if self.devlist == None: + self.devlist = self.db.get_refs('mds') + self.mdclist = [] self.desc_uuid = self.uuid self.uuid = uuid @@ -1785,18 +1789,16 @@ class MDSDEV(Module): manager.add_lustre_module('lov', 'lov') manager.add_lustre_module('mds', 'mds') - if self.fstype == 'smfs': - manager.add_lustre_module('smfs', 'smfs') - - if self.fstype == 'ldiskfs': - manager.add_lustre_module('ldiskfs', 'ldiskfs') - + if self.fstype == 'smfs' or self.fstype == 'ldiskfs': + manager.add_lustre_module(self.fstype, self.fstype) + if self.fstype: manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype)) # if fstype is smfs, then we should also take care about backing # store fs. if self.fstype == 'smfs': + manager.add_lustre_module(self.backfstype, self.backfstype) manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.backfstype)) for option in string.split(self.mountfsoptions, ','): @@ -1813,22 +1815,22 @@ class MDSDEV(Module): def get_mount_options(self, blkdev): options = def_mount_options(self.fstype, 'mds') - if config.mountfsoptions != None: - if options != None: + if config.mountfsoptions: + if options: options = "%s,%s" %(options, config.mountfsoptions) else: options = config.mountfsoptions - if self.mountfsoptions != None: + if self.mountfsoptions: options = "%s,%s" %(options, self.mountfsoptions) else: - if self.mountfsoptions != None: - if options != None: + if self.mountfsoptions: + if options: options = "%s,%s" %(options, self.mountfsoptions) else: options = self.mountfsoptions if self.fstype == 'smfs': - if options != None: + if options: options = "%s,type=%s,dev=%s" %(options, self.backfstype, blkdev) else: @@ -2105,14 +2107,14 @@ class OSD(Module): if self.active: manager.add_lustre_module('ost', 'ost') - if self.fstype == 'smfs': - manager.add_lustre_module('smfs', 'smfs') - - if self.fstype == 'ldiskfs': - manager.add_lustre_module('ldiskfs', 'ldiskfs') + if self.fstype == 'smfs' or self.fstype == 'ldiskfs': + manager.add_lustre_module(self.fstype, self.fstype) + if self.fstype: manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype)) + if self.fstype == 'smfs': + manager.add_lustre_module(self.backfstype, self.backfstype) manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.backfstype)) for option in self.mountfsoptions: @@ -2127,22 +2129,22 @@ class OSD(Module): def get_mount_options(self, blkdev): options = def_mount_options(self.fstype, 'ost') - if config.mountfsoptions != None: - if options != None: + if config.mountfsoptions: + if options: options = "%s,%s" %(options, config.mountfsoptions) else: options = config.mountfsoptions - if self.mountfsoptions != None: + if self.mountfsoptions: options = "%s,%s" %(options, self.mountfsoptions) else: - if self.mountfsoptions != None: - if options != None: + if self.mountfsoptions: + if options: options = "%s,%s" %(options, self.mountfsoptions) else: options = self.mountfsoptions if self.fstype == 'smfs': - if options != None: + if options: options = "%s,type=%s,dev=%s" %(options, self.backfstype, blkdev) else: @@ -2378,42 +2380,43 @@ class CMOBD(Module): cache_obd = self.db.lookup(self.cache_uuid) if not cache_obd: panic('cache obd not found:', self.cache_uuid) + + self.master = None + self.cache = None master_class = master_obd.get_class() cache_class = cache_obd.get_class() if master_class == 'ost' or master_class == 'lov': client_uuid = "%s_lov_master_UUID" % (self.name) - self.master = LOV(master_obd, client_uuid, self.name, - "%s_master" % (self.name)); - client_uuid = "%s_lov_cache_UUID" % (self.name) - self.cache = LOV(cache_obd, client_uuid, self.name, - "%s_cache" % (self.name)); - if master_class == 'mds': + self.master = LOV(master_obd, client_uuid, self.name); + elif master_class == 'mds': self.master = get_mdc(db, self.name, self.master_uuid) - if cache_class == 'mds': - self.cache = get_mdc(db, self.name, self.cache_uuid) - - if master_class == 'lmv': + elif master_class == 'lmv': client_uuid = "%s_lmv_master_UUID" % (self.name) - self.master = LMV(master_obd, client_uuid, self.name, - "%s_master" % (self.name)); - if cache_class == 'lmv': + self.master = LMV(master_obd, client_uuid, self.name); + else: + panic("unknown master obd class '%s'" %(master_class)) + + if cache_class == 'ost' or cache_class == 'lov': + client_uuid = "%s_lov_cache_UUID" % (self.name) + self.cache = LOV(cache_obd, client_uuid, self.name); + elif cache_class == 'mds': + self.cache = get_mdc(db, self.name, self.cache_uuid) + elif cache_class == 'lmv': client_uuid = "%s_lmv_cache_UUID" % (self.name) - self.cache = LMV(cache_obd, client_uuid, self.name, - "%s_cache" % (self.name)); + self.cache = LMV(cache_obd, client_uuid, self.name); + else: + panic("unknown cache obd class '%s'" %(cache_class)) - # need to check /proc/mounts and /etc/mtab before - # formatting anything. - # FIXME: check if device is already formatted. def prepare(self): self.master.prepare() if not config.record and is_prepared(self.name): return self.info(self.master_uuid, self.cache_uuid) lctl.newdev("cmobd", self.name, self.uuid, - setup ="%s %s" %(self.master_uuid, - self.cache_uuid)) + setup ="%s %s" %(self.master.uuid, + self.cache.uuid)) def get_uuid(self): return self.uuid @@ -2455,34 +2458,35 @@ class COBD(Module): cache_obd = self.db.lookup(self.cache_uuid) if not cache_obd: panic('cache obd not found:', self.cache_uuid) + + self.master = None + self.cache = None master_class = master_obd.get_class() cache_class = cache_obd.get_class() if master_class == 'ost' or master_class == 'lov': client_uuid = "%s_lov_master_UUID" % (self.name) - self.master = LOV(master_obd, client_uuid, name, - "%s_master" % (self.name)); - client_uuid = "%s_lov_cache_UUID" % (self.name) - self.cache = LOV(cache_obd, client_uuid, name, - "%s_cache" % (self.name)); - if master_class == 'mds': + self.master = LOV(master_obd, client_uuid, name); + elif master_class == 'mds': self.master = get_mdc(db, name, self.master_uuid) - if cache_class == 'mds': - self.cache = get_mdc(db, name, self.cache_uuid) - - if master_class == 'lmv': + elif master_class == 'lmv': client_uuid = "%s_lmv_master_UUID" % (self.name) - self.master = LMV(master_obd, client_uuid, self.name, - "%s_master" % (self.name)); - if cache_class == 'lmv': + self.master = LMV(master_obd, client_uuid, self.name); + else: + panic("unknown master obd class '%s'" %(master_class)) + + if cache_class == 'ost' or cache_class == 'lov': + client_uuid = "%s_lov_cache_UUID" % (self.name) + self.cache = LOV(cache_obd, client_uuid, name); + elif cache_class == 'mds': + self.cache = get_mdc(db, name, self.cache_uuid) + elif cache_class == 'lmv': client_uuid = "%s_lmv_cache_UUID" % (self.name) - self.cache = LMV(cache_obd, client_uuid, self.name, - "%s_cache" % (self.name)); + self.cache = LMV(cache_obd, client_uuid, self.name); + else: + panic("unknown cache obd class '%s'" %(cache_class)) - # need to check /proc/mounts and /etc/mtab before - # formatting anything. - # FIXME: check if device is already formatted. def get_uuid(self): return self.uuid diff --git a/lustre/utils/lmc b/lustre/utils/lmc index 84f4a06..700dc40 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -509,6 +509,11 @@ class GenConfig: lovconfig.appendChild(self.ref("lov", lov_uuid)) return lovconfig + def lmv_tgt(self, mdt_uuid): + tgt = self.doc.createElement('lmv_tgt') + tgt.setAttribute("uuidref", mdt_uuid) + return tgt + def lmv(self, name, uuid): lmv = self.newService("lmv", name, uuid) return lmv @@ -527,13 +532,13 @@ class GenConfig: mdd = self.newService("mdsdev", name, uuid) self.addElement(mdd, "fstype", fstype) if backfstype: - self.addElement(mdd, "backfstype", backfstype) + self.addElement(mdd, "backfstype", backfstype) dev = self.addElement(mdd, "devpath", devname) if backdevname: self.addElement(mdd, "backdevpath", backdevname) self.addElement(mdd, "autoformat", format) if dev_size: - self.addElement(mdd, "devsize", "%s" % (dev_size)) + self.addElement(mdd, "devsize", "%s" % (dev_size)) if journal_size: self.addElement(mdd, "journalsize", "%s" % (journal_size)) if inode_size: @@ -550,8 +555,11 @@ class GenConfig: self.addElement(mdd, "no_root_squash", no_root_squash) mdd.appendChild(self.ref("node", node_uuid)) mdd.appendChild(self.ref("target", mds_uuid)) - mdd.appendChild(self.dev(devname)) + dev = self.dev(devname) + if dev != None: + mdd.appendChild(dev) + if lmv_uuid: mdd.appendChild(self.ref("lmv", lmv_uuid)) self.addElement(mdd, "lmv", lmv_uuid) @@ -791,7 +799,7 @@ def lov_del_obd(gen, lustre, lov, osc_uuid, options): tgt.setAttribute('generation', str(genera)) def lmv_add_obd(gen, lmv, mdc_uuid): - lmv.appendChild(gen.ref("mds", mdc_uuid)) + lmv.appendChild(gen.lmv_tgt(mdc_uuid)) def ref_exists(profile, uuid): elist = profile.childNodes @@ -941,6 +949,8 @@ def add_route(gen, lustre, options): def add_mds(gen, lustre, options): node_name = get_option(options, 'node') mds_name = get_option(options, 'mds') + if not mds_name: + mds_name = new_name('MDS_'+ node_name) lmv_name = get_option(options, 'lmv') mdd_name = new_name("MDD_" + mds_name +"_" + node_name) mdd_uuid = new_uuid(mdd_name) @@ -1210,6 +1220,10 @@ def add_cobd(gen, lustre, options): master_uuid = name2uuid(lustre, master_name, tag='lov', fatal=0) cache_uuid = name2uuid(lustre, cache_name, tag='lov', fatal=0) + if not master_uuid or not cache_uuid: + master_uuid = name2uuid(lustre, master_name, tag='ost', fatal=0) + cache_uuid = name2uuid(lustre, cache_name, tag='ost', fatal=0) + if master_uuid: node = lookup(lustre, master_uuid) rets = node.getElementsByTagName('lov_tgt') @@ -1364,11 +1378,27 @@ def add_lmv(gen, lustre, options): lmv = gen.lmv(name, uuid) lustre.appendChild(lmv) - + +def find_client(lustre, mds_uuid, client_uuid): + mds = lookup(lustre, mds_uuid) + clients = mds.getElementsByTagName('client_ref') + + if clients: + for client in clients: + if client.getAttribute("uuidref") == client_uuid: + return 1 + return 0 + def new_filesystem(gen, lustre, mds_uuid, obd_uuid, mgmt_uuid): fs_name = new_name("FS_fsname") fs_uuid = new_uuid(fs_name) + mds = lookup(lustre, mds_uuid) + clients = mds.getElementsByTagName('client_ref') + + if find_client(lustre, mds_uuid, obd_uuid) == 0: + mds.appendChild(gen.ref("client", obd_uuid)) + fs = gen.filesystem(fs_name, fs_uuid, mds_uuid, obd_uuid, mgmt_uuid) @@ -1663,8 +1693,8 @@ def main(): try: options, args = cl.parse(cmdlinesplit(cmd)) if options.merge or options.input or options.output: - print "The batchfile should not contain --merge, --input or --output." - sys.exit(1) + print "The batchfile should not contain --merge, --input or --output." + sys.exit(1) do_command(gen, lustre, options, args) except OptionError, e: panic(cmd, e) -- 1.8.3.1