From d43cd3c715f777cc69c186acee51c9f6391be6ff Mon Sep 17 00:00:00 2001 From: nathan Date: Fri, 1 Apr 2005 19:36:38 +0000 Subject: [PATCH] b=3026 r=boc support for arbitrary number of OSTs --- lustre/ChangeLog | 3 +- lustre/include/linux/lustre_cfg.h | 2 + lustre/include/linux/lustre_ha.h | 1 + lustre/include/linux/obd.h | 3 + lustre/include/linux/obd_class.h | 13 + lustre/include/lustre/lustre_user.h | 7 +- lustre/lov/lov_log.c | 26 +- lustre/lov/lov_obd.c | 553 +++++++++++++++++++++++++++--------- lustre/obdclass/lprocfs_status.c | 1 + lustre/obdclass/obd_config.c | 8 +- lustre/ptlrpc/import.c | 4 +- lustre/ptlrpc/ptlrpc_module.c | 1 + lustre/utils/Lustre/lustredb.py | 17 ++ lustre/utils/lconf | 101 +++++-- lustre/utils/lctl.c | 10 +- lustre/utils/liblustreapi.c | 13 +- lustre/utils/lustre_cfg.c | 160 +++++++---- lustre/utils/obd.c | 35 ++- lustre/utils/obdctl.h | 1 + 19 files changed, 704 insertions(+), 255 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index d39c02c..7068ead 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -26,7 +26,8 @@ servers (3262) - add --abort-recovery option to lconf to abort recovery on device startup (6017) - + - add support for an arbitrary number of OSTs (3026) + 2005-03-22 Cluster File Systems, Inc. * version 1.4.1 * bug fixes diff --git a/lustre/include/linux/lustre_cfg.h b/lustre/include/linux/lustre_cfg.h index d1459da..bfc66a4 100644 --- a/lustre/include/linux/lustre_cfg.h +++ b/lustre/include/linux/lustre_cfg.h @@ -46,6 +46,8 @@ enum lcfg_command_type { LCFG_SET_UPCALL = 0x00cf00a, LCFG_ADD_CONN = 0x00cf00b, LCFG_DEL_CONN = 0x00cf00c, + LCFG_LOV_ADD_OBD = 0x00cf00d, + LCFG_LOV_DEL_OBD = 0x00cf00e }; struct lustre_cfg_bufs { diff --git a/lustre/include/linux/lustre_ha.h b/lustre/include/linux/lustre_ha.h index eb44bbe..8b7bef4 100644 --- a/lustre/include/linux/lustre_ha.h +++ b/lustre/include/linux/lustre_ha.h @@ -19,6 +19,7 @@ void ptlrpc_free_committed(struct obd_import *imp); void ptlrpc_wake_delayed(struct obd_import *imp); int ptlrpc_recover_import(struct obd_import *imp, char *new_uuid); int ptlrpc_set_import_active(struct obd_import *imp, int active); +void ptlrpc_activate_import(struct obd_import *imp); void ptlrpc_deactivate_import(struct obd_import *imp); void ptlrpc_invalidate_import(struct obd_import *imp); void ptlrpc_fail_import(struct obd_import *imp, int generation); diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index 4d74a59..0ac8a9e 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -388,6 +388,7 @@ struct cache_obd { struct lov_tgt_desc { struct obd_uuid uuid; + __u32 ltd_gen; struct obd_export *ltd_exp; int active; /* is this target up for requests */ }; @@ -564,6 +565,8 @@ struct obd_ops { int (*o_setup) (struct obd_device *dev, obd_count len, void *data); int (*o_precleanup)(struct obd_device *dev); int (*o_cleanup)(struct obd_device *dev); + int (*o_process_config)(struct obd_device *dev, obd_count len, + void *data); int (*o_postrecov)(struct obd_device *dev); int (*o_add_conn)(struct obd_import *imp, struct obd_uuid *uuid, int priority); diff --git a/lustre/include/linux/obd_class.h b/lustre/include/linux/obd_class.h index 1e3dade..960fbe5 100644 --- a/lustre/include/linux/obd_class.h +++ b/lustre/include/linux/obd_class.h @@ -302,6 +302,19 @@ static inline int obd_cleanup(struct obd_device *obd) RETURN(rc); } +static inline int +obd_process_config(struct obd_device *obd, int datalen, void *data) +{ + int rc; + ENTRY; + + OBD_CHECK_OP(obd, process_config, -EOPNOTSUPP); + OBD_COUNTER_INCREMENT(obd, process_config); + + rc = OBP(obd, process_config)(obd, datalen, data); + RETURN(rc); +} + /* Pack an in-memory MD struct for storage on disk. * Returns +ve size of packed MD (0 for free), or -ve error. * diff --git a/lustre/include/lustre/lustre_user.h b/lustre/include/lustre/lustre_user.h index fe82883..9de6b72 100644 --- a/lustre/include/lustre/lustre_user.h +++ b/lustre/include/lustre/lustre_user.h @@ -56,7 +56,7 @@ #define lov_user_ost_data lov_user_ost_data_v1 struct lov_user_ost_data_v1 { /* per-stripe data structure */ - __u64 l_object_id; /* OST object ID */ + __u64 l_object_id; /* OST object ID */ __u64 l_object_gr; /* OST object group (creating MDS number) */ __u32 l_ost_gen; /* generation of this OST index */ __u32 l_ost_idx; /* OST index in LOV */ @@ -107,6 +107,11 @@ static inline int obd_uuid_equals(struct obd_uuid *u1, struct obd_uuid *u2) return strcmp((char *)u1->uuid, (char *)u2->uuid) == 0; } +static inline int obd_uuid_empty(struct obd_uuid *uuid) +{ + return uuid->uuid[0] == '\0'; +} + static inline void obd_str2uuid(struct obd_uuid *uuid, char *tmp) { strncpy((char *)uuid->uuid, tmp, sizeof(*uuid)); diff --git a/lustre/lov/lov_log.c b/lustre/lov/lov_log.c index f04c996..7a52d72 100644 --- a/lustre/lov/lov_log.c +++ b/lustre/lov/lov_log.c @@ -97,14 +97,19 @@ static int lov_llog_origin_connect(struct llog_ctxt *ctxt, int count, { struct obd_device *obd = ctxt->loc_obd; struct lov_obd *lov = &obd->u.lov; + struct lov_tgt_desc *tgt; int i, rc = 0; ENTRY; LASSERT(lov->desc.ld_tgt_count == count); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - struct obd_device *child = lov->tgts[i].ltd_exp->exp_obd; - struct llog_ctxt *cctxt = llog_get_context(child, ctxt->loc_idx); - + for (i = 0, tgt = lov->tgts; i < lov->desc.ld_tgt_count; i++, tgt++) { + struct obd_device *child; + struct llog_ctxt *cctxt; + + if (!tgt->active) + continue; + child = tgt->ltd_exp->exp_obd; + cctxt = llog_get_context(child, ctxt->loc_idx); if (uuid && !obd_uuid_equals(uuid, &lov->tgts[i].uuid)) continue; @@ -163,6 +168,7 @@ int lov_llog_init(struct obd_device *obd, struct obd_device *tgt, int count, struct llog_catid *logid) { struct lov_obd *lov = &obd->u.lov; + struct lov_tgt_desc *ctgt; int i, rc = 0; ENTRY; @@ -176,9 +182,12 @@ int lov_llog_init(struct obd_device *obd, struct obd_device *tgt, if (rc) RETURN(rc); - LASSERT(lov->desc.ld_tgt_count == count); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - struct obd_device *child = lov->tgts[i].ltd_exp->exp_obd; + LASSERT(lov->desc.ld_tgt_count == count); + for (i = 0, ctgt = lov->tgts; i < lov->desc.ld_tgt_count; i++, ctgt++) { + struct obd_device *child; + if (!ctgt->active) + continue; + child = ctgt->ltd_exp->exp_obd; rc = obd_llog_init(child, tgt, 1, logid + i); if (rc) { CERROR("error osc_llog_init %d\n", i); @@ -195,7 +204,7 @@ int lov_llog_finish(struct obd_device *obd, int count) ENTRY; /* cleanup our llogs only if the ctxts have been setup - * (lov1 doesn't setup, lov_mds1 does). */ + * (client lov doesn't setup, mds lov does). */ ctxt = llog_get_context(obd, LLOG_UNLINK_ORIG_CTXT); if (ctxt) rc = llog_cleanup(ctxt); @@ -206,5 +215,6 @@ int lov_llog_finish(struct obd_device *obd, int count) if (!rc) rc = rc2; + /* lov->tgt llogs are cleaned during osc_cleanup. */ RETURN(rc); } diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index a7517fa..05bc98c 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -53,13 +53,100 @@ #include "lov_internal.h" /* obd methods */ +#define MAX_STRING_SIZE 128 +static int lov_connect_obd(struct obd_device *obd, struct lov_tgt_desc *tgt, + int activate, struct obd_connect_data *data) +{ + struct lov_obd *lov = &obd->u.lov; + struct obd_uuid *tgt_uuid = &tgt->uuid; + struct obd_device *tgt_obd; + struct obd_uuid lov_osc_uuid = { "LOV_OSC_UUID" }; + struct lustre_handle conn = {0, }; + struct proc_dir_entry *lov_proc_dir; + int rc; + ENTRY; + + tgt_obd = class_find_client_obd(tgt_uuid, LUSTRE_OSC_NAME, + &obd->obd_uuid); + + if (!tgt_obd) { + CERROR("Target %s not attached\n", tgt_uuid->uuid); + RETURN(-EINVAL); + } + + if (!tgt_obd->obd_set_up) { + CERROR("Target %s not set up\n", tgt_uuid->uuid); + RETURN(-EINVAL); + } + + if (activate) { + tgt_obd->obd_no_recov = 0; + ptlrpc_activate_import(tgt_obd->u.cli.cl_import); + } + + if (tgt_obd->u.cli.cl_import->imp_invalid) { + CERROR("not connecting OSC %s; administratively " + "disabled\n", tgt_uuid->uuid); + rc = obd_register_observer(tgt_obd, obd); + if (rc) { + CERROR("Target %s register_observer error %d; " + "will not be able to reactivate\n", + tgt_uuid->uuid, rc); + } + RETURN(0); + } + + rc = obd_connect(&conn, tgt_obd, &lov_osc_uuid, data); + if (rc) { + CERROR("Target %s connect error %d\n", tgt_uuid->uuid, rc); + RETURN(rc); + } + tgt->ltd_exp = class_conn2export(&conn); + + rc = obd_register_observer(tgt_obd, obd); + if (rc) { + CERROR("Target %s register_observer error %d\n", + tgt_uuid->uuid, rc); + obd_disconnect(tgt->ltd_exp); + tgt->ltd_exp = NULL; + RETURN(rc); + } + + tgt->active = 1; + lov->desc.ld_active_tgt_count++; + + lov_proc_dir = lprocfs_srch(obd->obd_proc_entry, "target_obds"); + if (lov_proc_dir) { + struct obd_device *osc_obd = class_conn2obd(&conn); + struct proc_dir_entry *osc_symlink; + char name[MAX_STRING_SIZE]; + + LASSERT(osc_obd != NULL); + LASSERT(osc_obd->obd_type != NULL); + LASSERT(osc_obd->obd_type->typ_name != NULL); + snprintf(name, MAX_STRING_SIZE, "../../../%s/%s", + osc_obd->obd_type->typ_name, + osc_obd->obd_name); + osc_symlink = proc_symlink(osc_obd->obd_name, lov_proc_dir, + name); + if (osc_symlink == NULL) { + CERROR("could not register LOV target " + "/proc/fs/lustre/%s/%s/target_obds/%s.", + obd->obd_type->typ_name, obd->obd_name, + osc_obd->obd_name); + lprocfs_remove(lov_proc_dir); + lov_proc_dir = NULL; + } + } + + RETURN(0); +} + static int lov_connect(struct lustre_handle *conn, struct obd_device *obd, struct obd_uuid *cluuid, struct obd_connect_data *data) { - struct ptlrpc_request *req = NULL; struct lov_obd *lov = &obd->u.lov; - struct lov_desc *desc = &lov->desc; - struct lov_tgt_desc *tgts; + struct lov_tgt_desc *tgt; struct obd_export *exp; int rc, rc2, i; ENTRY; @@ -78,72 +165,26 @@ static int lov_connect(struct lustre_handle *conn, struct obd_device *obd, RETURN(0); } - for (i = 0, tgts = lov->tgts; i < desc->ld_tgt_count; i++, tgts++) { - struct obd_uuid *tgt_uuid = &tgts->uuid; - struct obd_device *tgt_obd; - struct obd_uuid lov_osc_uuid = { "LOV_OSC_UUID" }; - struct lustre_handle conn = {0, }; - - LASSERT( tgt_uuid != NULL); - - tgt_obd = class_find_client_obd(tgt_uuid, LUSTRE_OSC_NAME, - &obd->obd_uuid); - - if (!tgt_obd) { - CERROR("Target %s not attached\n", tgt_uuid->uuid); - GOTO(out_disc, rc = -EINVAL); - } - - if (!tgt_obd->obd_set_up) { - CERROR("Target %s not set up\n", tgt_uuid->uuid); - GOTO(out_disc, rc = -EINVAL); - } - - if (tgt_obd->u.cli.cl_import->imp_invalid) { - CERROR("not connecting OSC %s; administratively " - "disabled\n", tgt_uuid->uuid); - rc = obd_register_observer(tgt_obd, obd); - if (rc) { - CERROR("Target %s register_observer error %d; " - "will not be able to reactivate\n", - tgt_uuid->uuid, rc); - } + for (i = 0, tgt = lov->tgts; i < lov->desc.ld_tgt_count; i++, tgt++) { + if (obd_uuid_empty(&tgt->uuid)) continue; - } - - rc = obd_connect(&conn, tgt_obd, &lov_osc_uuid, data); - if (rc) { - CERROR("Target %s connect error %d\n", tgt_uuid->uuid, - rc); - GOTO(out_disc, rc); - } - tgts->ltd_exp = class_conn2export(&conn); - - rc = obd_register_observer(tgt_obd, obd); - if (rc) { - CERROR("Target %s register_observer error %d\n", - tgt_uuid->uuid, rc); - obd_disconnect(tgts->ltd_exp); + rc = lov_connect_obd(obd, tgt, 0, data); + if (rc) GOTO(out_disc, rc); - } - - desc->ld_active_tgt_count++; - tgts->active = 1; } - - ptlrpc_req_finished(req); + class_export_put(exp); RETURN (0); out_disc: while (i-- > 0) { struct obd_uuid uuid; - --tgts; - --desc->ld_active_tgt_count; - tgts->active = 0; + --tgt; + --lov->desc.ld_active_tgt_count; + tgt->active = 0; /* save for CERROR below; (we know it's terminated) */ - uuid = tgts->uuid; - rc2 = obd_disconnect(tgts->ltd_exp); + uuid = tgt->uuid; + rc2 = obd_disconnect(tgt->ltd_exp); if (rc2) CERROR("error: LOV target %s disconnect on OST idx %d: " "rc = %d\n", uuid.uuid, i, rc2); @@ -152,11 +193,62 @@ static int lov_connect(struct lustre_handle *conn, struct obd_device *obd, RETURN (rc); } +static int lov_disconnect_obd(struct obd_device *obd, struct lov_tgt_desc *tgt) +{ + struct proc_dir_entry *lov_proc_dir; + struct obd_device *osc_obd = class_exp2obd(tgt->ltd_exp); + struct lov_obd *lov = &obd->u.lov; + int rc; + ENTRY; + + lov_proc_dir = lprocfs_srch(obd->obd_proc_entry, "target_obds"); + if (lov_proc_dir) { + struct proc_dir_entry *osc_symlink; + + osc_symlink = lprocfs_srch(lov_proc_dir, osc_obd->obd_name); + if (osc_symlink) { + lprocfs_remove(osc_symlink); + } else { + CERROR("/proc/fs/lustre/%s/%s/target_obds/%s missing.", + obd->obd_type->typ_name, obd->obd_name, + osc_obd->obd_name); + } + } + + if (obd->obd_no_recov) { + /* Pass it on to our clients. + * XXX This should be an argument to disconnect, + * XXX not a back-door flag on the OBD. Ah well. + */ + if (osc_obd) + osc_obd->obd_no_recov = 1; + } + + obd_register_observer(tgt->ltd_exp->exp_obd, NULL); + + rc = obd_disconnect(tgt->ltd_exp); + if (rc) { + if (tgt->active) { + CERROR("Target %s disconnect error %d\n", + tgt->uuid.uuid, rc); + } + rc = 0; + } + + if (tgt->active) { + tgt->active = 0; + lov->desc.ld_active_tgt_count--; + } + + tgt->ltd_exp = NULL; + RETURN(0); +} + static int lov_disconnect(struct obd_export *exp) { struct obd_device *obd = class_exp2obd(exp); struct lov_obd *lov = &obd->u.lov; - struct obd_export *osc_exp; + struct lov_tgt_desc *tgt; int rc, i; ENTRY; @@ -168,43 +260,10 @@ static int lov_disconnect(struct obd_export *exp) if (lov->refcount != 0) goto out_local; - spin_lock(&lov->lov_lock); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - if (lov->tgts[i].ltd_exp == NULL) - continue; - - osc_exp = lov->tgts[i].ltd_exp; - lov->tgts[i].ltd_exp = NULL; - - if (obd->obd_no_recov) { - /* Pass it on to our clients. - * XXX This should be an argument to disconnect, - * XXX not a back-door flag on the OBD. Ah well. - */ - struct obd_device *osc_obd; - osc_obd = class_exp2obd(osc_exp); - if (osc_obd) - osc_obd->obd_no_recov = 1; - } - - obd_register_observer(osc_exp->exp_obd, NULL); - - spin_unlock(&lov->lov_lock); - rc = obd_disconnect(osc_exp); - spin_lock(&lov->lov_lock); - if (rc) { - if (lov->tgts[i].active) { - CERROR("Target %s disconnect error %d\n", - lov->tgts[i].uuid.uuid, rc); - } - rc = 0; - } - if (lov->tgts[i].active) { - lov->desc.ld_active_tgt_count--; - lov->tgts[i].active = 0; - } + for (i = 0, tgt = lov->tgts; i < lov->desc.ld_tgt_count; i++, tgt++) { + if (tgt->ltd_exp) + lov_disconnect_obd(obd, tgt); } - spin_unlock(&lov->lov_lock); out_local: rc = class_disconnect(exp); @@ -299,9 +358,6 @@ static int lov_setup(struct obd_device *obd, obd_count len, void *buf) struct lustre_cfg *lcfg = buf; struct lov_desc *desc; struct lov_obd *lov = &obd->u.lov; - struct obd_uuid *uuids; - struct lov_tgt_desc *tgts; - int i; int count; ENTRY; @@ -310,12 +366,8 @@ static int lov_setup(struct obd_device *obd, obd_count len, void *buf) RETURN(-EINVAL); } - if (LUSTRE_CFG_BUFLEN(lcfg, 2) < 1) { - CERROR("LOV setup requires an OST UUID list\n"); - RETURN(-EINVAL); - } - desc = (struct lov_desc *)lustre_cfg_buf(lcfg, 1); + if (sizeof(*desc) > LUSTRE_CFG_BUFLEN(lcfg, 1)) { CERROR("descriptor size wrong: %d > %d\n", (int)sizeof(*desc), LUSTRE_CFG_BUFLEN(lcfg, 1)); @@ -334,16 +386,6 @@ static int lov_setup(struct obd_device *obd, obd_count len, void *buf) } } - desc->ld_active_tgt_count = 0; - count = desc->ld_tgt_count; - uuids = (struct obd_uuid *)lustre_cfg_buf(lcfg, 2); - if (sizeof(*uuids) * count != LUSTRE_CFG_BUFLEN(lcfg, 2)) { - CERROR("UUID array size wrong: %u * %u != %u\n", - (int)sizeof(*uuids), count, - LUSTRE_CFG_BUFLEN(lcfg, 2)); - RETURN(-EINVAL); - } - if (desc->ld_default_stripe_size < PTLRPC_MAX_BRW_SIZE) { CWARN("Increasing default_stripe_size "LPU64" to %u\n", desc->ld_default_stripe_size, PTLRPC_MAX_BRW_SIZE); @@ -356,33 +398,28 @@ static int lov_setup(struct obd_device *obd, obd_count len, void *buf) * divisor in a 32-bit kernel, we cannot support a stripe width * of 4GB or larger on 32-bit CPUs. */ - if ((desc->ld_default_stripe_count ? - desc->ld_default_stripe_count : desc->ld_tgt_count) * - desc->ld_default_stripe_size > ~0UL) { + count = desc->ld_default_stripe_count; + if (count && (count * desc->ld_default_stripe_size) > ~0UL) { CERROR("LOV: stripe width "LPU64"x%u > %lu on 32-bit system\n", - desc->ld_default_stripe_size, - desc->ld_default_stripe_count ? - desc->ld_default_stripe_count : desc->ld_tgt_count,~0UL); + desc->ld_default_stripe_size, count, ~0UL); RETURN(-EINVAL); } - + + /* Allocate space for target list */ + if (desc->ld_tgt_count) + count = desc->ld_tgt_count; lov->bufsize = sizeof(struct lov_tgt_desc) * count; OBD_ALLOC(lov->tgts, lov->bufsize); if (lov->tgts == NULL) { CERROR("Out of memory\n"); RETURN(-EINVAL); } + memset(lov->tgts, 0, lov->bufsize); + desc->ld_active_tgt_count = 0; lov->desc = *desc; spin_lock_init(&lov->lov_lock); - - for (i = 0, tgts = lov->tgts; i < desc->ld_tgt_count; i++, tgts++) { - struct obd_uuid *uuid = &tgts->uuid; - - /* NULL termination already checked */ - *uuid = uuids[i]; - } - + lprocfs_init_vars(lov, &lvars); lprocfs_obd_setup(obd, lvars.obd_vars); #ifdef __KERNEL__ @@ -407,11 +444,204 @@ static int lov_cleanup(struct obd_device *obd) lprocfs_obd_cleanup(obd); obd_llog_finish(obd, 0); - OBD_FREE(lov->tgts, lov->bufsize); + if (lov->tgts) + OBD_FREE(lov->tgts, lov->bufsize); RETURN(0); } +static int +lov_add_obd(struct obd_device *obd, struct obd_uuid *uuidp, int index, int gen) +{ + struct lov_obd *lov = &obd->u.lov; + struct lov_tgt_desc *tgt; + struct obd_export *exp_observer; + __u32 bufsize; + __u32 size = 2; + obd_id params[2]; + int rc, old_count; + ENTRY; + + CDEBUG(D_CONFIG, "uuid: %s idx: %d gen: %d\n", + uuidp->uuid, index, gen); + + if (index < 0) { + CERROR("request to add OBD %s at invalid index: %d\n", + uuidp->uuid, index); + RETURN(-EINVAL); + } + + if (gen <= 0) { + CERROR("request to add OBD %s with invalid generation: %d\n", + uuidp->uuid, gen); + RETURN(-EINVAL); + } + + bufsize = sizeof(struct lov_tgt_desc) * (index + 1); + if (bufsize > lov->bufsize) { + OBD_ALLOC(tgt, bufsize); + if (tgt == NULL) { + CERROR("couldn't allocate %d bytes for new table.\n", + bufsize); + RETURN(-ENOMEM); + } + + memset(tgt, 0, bufsize); + if (lov->tgts) { + memcpy(tgt, lov->tgts, lov->bufsize); + OBD_FREE(lov->tgts, lov->bufsize); + } + + lov->tgts = tgt; + lov->bufsize = bufsize; + CDEBUG(D_CONFIG, "tgts: %p bufsize: %d\n", + lov->tgts, lov->bufsize); + } + + tgt = &lov->tgts[index]; + if (!obd_uuid_empty(&tgt->uuid)) { + CERROR("OBD already assigned at LOV target index %d\n", + index); + RETURN(-EEXIST); + } + + tgt->uuid = *uuidp; + /* XXX - add a sanity check on the generation number. */ + tgt->ltd_gen = gen; + + old_count = lov->desc.ld_tgt_count; + if (index >= lov->desc.ld_tgt_count) + lov->desc.ld_tgt_count = index + 1; + + CDEBUG(D_CONFIG, "idx=%d ltd_gen=%d ld_tgt_count=%d\n", + index, tgt->ltd_gen, lov->desc.ld_tgt_count); + + if (lov->refcount == 0) + /* lov_connect hasn't been called yet. So we'll do the + lov_connect_obd on this obd when that fn first runs. */ + RETURN(0); + + if (tgt->ltd_exp) { + struct obd_device *osc_obd; + + osc_obd = class_exp2obd(tgt->ltd_exp); + if (osc_obd) + osc_obd->obd_no_recov = 0; + } + + /* NULL may need to change when we use flags for osc's */ + rc = lov_connect_obd(obd, tgt, 1, NULL); + if (rc || !obd->obd_observer) + RETURN(rc); + + /* tell the mds_lov about the new target */ + obd_llog_finish(obd->obd_observer, old_count); + llog_cat_initialize(obd->obd_observer, lov->desc.ld_tgt_count); + + params[0] = index; + rc = obd_get_info(tgt->ltd_exp, strlen("last_id"), "last_id", &size, + ¶ms[1]); + if (rc) + GOTO(out, rc); + + exp_observer = obd->obd_observer->obd_self_export; + rc = obd_set_info(exp_observer, strlen("next_id"),"next_id", 2, params); + if (rc) + GOTO(out, rc); + + rc = lov_notify(obd, tgt->ltd_exp->exp_obd, 1); + GOTO(out, rc); + out: + if (rc && tgt->ltd_exp != NULL) + lov_disconnect_obd(obd, tgt); + return rc; +} + +static int +lov_del_obd(struct obd_device *obd, struct obd_uuid *uuidp, int index, int gen) +{ + struct lov_obd *lov = &obd->u.lov; + struct lov_tgt_desc *tgt; + int count = lov->desc.ld_tgt_count; + int rc = 0; + ENTRY; + + CDEBUG(D_CONFIG, "uuid: %s idx: %d gen: %d\n", + uuidp->uuid, index, gen); + + if (index >= count) { + CERROR("LOV target index %d >= number of LOV OBDs %d.\n", + index, count); + RETURN(-EINVAL); + } + + tgt = lov->tgts + index; + + if (obd_uuid_empty(&tgt->uuid)) { + CERROR("LOV target at index %d is not setup.\n", index); + RETURN(-EINVAL); + } + + if (strncmp(uuidp->uuid, tgt->uuid.uuid, sizeof uuidp->uuid) != 0) { + CERROR("LOV target UUID %s at index %d doesn't match %s.\n", + tgt->uuid.uuid, index, uuidp->uuid); + RETURN(-EINVAL); + } + + if (tgt->ltd_exp) + lov_disconnect_obd(obd, tgt); + + /* XXX - right now there is a dependency on ld_tgt_count being the + * maximum tgt index for computing the mds_max_easize. So we can't + * shrink it. */ + + /* lt_gen = 0 will mean it will not match the gen of any valid loi */ + memset(tgt, 0, sizeof(*tgt)); + + CDEBUG(D_CONFIG, "uuid: %s idx: %d gen: %d exp: %p active: %d\n", + tgt->uuid.uuid, index, tgt->ltd_gen, tgt->ltd_exp, tgt->active); + + RETURN(rc); +} + +static int lov_process_config(struct obd_device *obd, obd_count len, void *buf) +{ + struct lustre_cfg *lcfg = buf; + struct obd_uuid obd_uuid; + int cmd; + int index; + int gen; + int rc = 0; + ENTRY; + + switch(cmd = lcfg->lcfg_command) { + case LCFG_LOV_ADD_OBD: + case LCFG_LOV_DEL_OBD: { + if (LUSTRE_CFG_BUFLEN(lcfg, 1) > sizeof(obd_uuid.uuid)) + GOTO(out, rc = -EINVAL); + + obd_str2uuid(&obd_uuid, lustre_cfg_buf(lcfg, 1)); + + if (sscanf(lustre_cfg_buf(lcfg, 2), "%d", &index) != 1) + GOTO(out, rc = -EINVAL); + if (sscanf(lustre_cfg_buf(lcfg, 3), "%d", &gen) != 1) + GOTO(out, rc = -EINVAL); + if (cmd == LCFG_LOV_ADD_OBD) + rc = lov_add_obd(obd, &obd_uuid, index, gen); + else + rc = lov_del_obd(obd, &obd_uuid, index, gen); + GOTO(out, rc); + } + default: { + CERROR("Unknown command: %d\n", lcfg->lcfg_command); + GOTO(out, rc = -EINVAL); + + } + } +out: + RETURN(rc); +} + #ifndef log2 #define log2(n) ffz(~(n)) #endif @@ -1091,6 +1321,7 @@ static int lov_queue_async_io(struct obd_export *exp, lap = LAP_FROM_COOKIE(cookie); loi = &lsm->lsm_oinfo[lap->lap_stripe]; + rc = obd_queue_async_io(lov->tgts[loi->loi_ost_idx].ltd_exp, lsm, loi, lap->lap_sub_cookie, cmd, off, count, brw_flags, async_flags); @@ -1113,6 +1344,7 @@ static int lov_set_async_flags(struct obd_export *exp, lap = LAP_FROM_COOKIE(cookie); loi = &lsm->lsm_oinfo[lap->lap_stripe]; + rc = obd_set_async_flags(lov->tgts[loi->loi_ost_idx].ltd_exp, lsm, loi, lap->lap_sub_cookie, async_flags); RETURN(rc); @@ -1136,6 +1368,7 @@ static int lov_queue_group_io(struct obd_export *exp, lap = LAP_FROM_COOKIE(cookie); loi = &lsm->lsm_oinfo[lap->lap_stripe]; + rc = obd_queue_group_io(lov->tgts[loi->loi_ost_idx].ltd_exp, lsm, loi, oig, lap->lap_sub_cookie, cmd, off, count, brw_flags, async_flags); @@ -1157,8 +1390,13 @@ static int lov_trigger_group_io(struct obd_export *exp, ASSERT_LSM_MAGIC(lsm); - for (i = 0, loi = lsm->lsm_oinfo; i < lsm->lsm_stripe_count; - i++, loi++) { + loi = lsm->lsm_oinfo; + for (i = 0; i < lsm->lsm_stripe_count; i++, loi++) { + if (lov->tgts[loi->loi_ost_idx].active == 0) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } + err = obd_trigger_group_io(lov->tgts[loi->loi_ost_idx].ltd_exp, lsm, loi, oig); if (rc == 0 && err != 0) @@ -1182,6 +1420,7 @@ static int lov_teardown_async_page(struct obd_export *exp, lap = LAP_FROM_COOKIE(cookie); loi = &lsm->lsm_oinfo[lap->lap_stripe]; + rc = obd_teardown_async_page(lov->tgts[loi->loi_ost_idx].ltd_exp, lsm, loi, lap->lap_sub_cookie); if (rc) { @@ -1303,8 +1542,10 @@ static int lov_change_cbdata(struct obd_export *exp, lov = &exp->exp_obd->u.lov; for (i = 0,loi = lsm->lsm_oinfo; i < lsm->lsm_stripe_count; i++,loi++) { struct lov_stripe_md submd; - if (lov->tgts[loi->loi_ost_idx].active == 0) + if (lov->tgts[loi->loi_ost_idx].active == 0) { CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } submd.lsm_object_id = loi->loi_id; submd.lsm_stripe_count = 0; @@ -1364,12 +1605,22 @@ static int lov_cancel_unused(struct obd_export *exp, int rc = 0, i; ENTRY; + lov = &exp->exp_obd->u.lov; + if (lsm == NULL) { + for (i = 0; i < lov->desc.ld_tgt_count; i++) { + int err = obd_cancel_unused(lov->tgts[i].ltd_exp, NULL, + flags, opaque); + if (!rc) + rc = err; + } + RETURN(rc); + } + ASSERT_LSM_MAGIC(lsm); if (!exp || !exp->exp_obd) RETURN(-ENODEV); - lov = &exp->exp_obd->u.lov; for (i = 0,loi = lsm->lsm_oinfo; i < lsm->lsm_stripe_count; i++,loi++) { struct lov_stripe_md submd; int err; @@ -1451,7 +1702,6 @@ static int lov_statfs(struct obd_device *obd, struct obd_statfs *osfs, /* We only get block data from the OBD */ for (i = 0; i < lov->desc.ld_tgt_count; i++) { int err; - if (!lov->tgts[i].active) { CDEBUG(D_HA, "lov idx %d inactive\n", i); continue; @@ -1508,10 +1758,8 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, { struct obd_device *obddev = class_exp2obd(exp); struct lov_obd *lov = &obddev->u.lov; - int i, count = lov->desc.ld_tgt_count; + int i, rc, count = lov->desc.ld_tgt_count; struct obd_uuid *uuidp; - int rc; - ENTRY; switch (cmd) { @@ -1520,6 +1768,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, struct lov_tgt_desc *tgtdesc; struct lov_desc *desc; char *buf = NULL; + __u32 *genp; buf = NULL; len = 0; @@ -1538,13 +1787,22 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, RETURN(-EINVAL); } + if (sizeof(__u32) * count > data->ioc_inllen3) { + OBD_FREE(buf, len); + RETURN(-EINVAL); + } + desc = (struct lov_desc *)data->ioc_inlbuf1; memcpy(desc, &(lov->desc), sizeof(*desc)); uuidp = (struct obd_uuid *)data->ioc_inlbuf2; + genp = (__u32 *)data->ioc_inlbuf3; tgtdesc = lov->tgts; - for (i = 0; i < count; i++, uuidp++, tgtdesc++) + /* the uuid will be empty for deleted OSTs */ + for (i = 0; i < count; i++, uuidp++, genp++, tgtdesc++) { obd_str2uuid(uuidp, tgtdesc->uuid.uuid); + *genp = tgtdesc->ltd_gen; + } rc = copy_to_user((void *)uarg, buf, len); if (rc) @@ -1569,6 +1827,10 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, for (i = 0; i < count; i++) { int err; + /* OST was deleted */ + if (obd_uuid_empty(&lov->tgts[i].uuid)) + continue; + err = obd_iocontrol(cmd, lov->tgts[i].ltd_exp, len, karg, uarg); if (err) { @@ -1619,6 +1881,9 @@ static int lov_get_info(struct obd_export *exp, __u32 keylen, /* XXX This is another one of those bits that will need to * change if we ever actually support nested LOVs. It uses * the lock's export to find out which stripe it is. */ + /* XXX - it's assumed all the locks for deleted OSTs have + * been cancelled. Also, the export for deleted OSTs will + * be NULL and won't match the lock's export. */ for (i = 0, loi = data->lsm->lsm_oinfo; i < data->lsm->lsm_stripe_count; i++, loi++) { @@ -1682,6 +1947,10 @@ static int lov_set_info(struct obd_export *exp, obd_count keylen, if (vallen != lov->desc.ld_tgt_count) RETURN(-EINVAL); for (i = 0; i < lov->desc.ld_tgt_count; i++) { + /* OST was deleted */ + if (obd_uuid_empty(&lov->tgts[i].uuid)) + continue; + /* initialize all OSCs, even inactive ones */ err = obd_set_info(lov->tgts[i].ltd_exp, keylen, key, sizeof(obd_id), @@ -1703,6 +1972,10 @@ static int lov_set_info(struct obd_export *exp, obd_count keylen, if (val && !obd_uuid_equals(val, &lov->tgts[i].uuid)) continue; + /* OST was deleted */ + if (obd_uuid_empty(&lov->tgts[i].uuid)) + continue; + if (!val && !lov->tgts[i].active) continue; @@ -1824,6 +2097,7 @@ struct obd_ops lov_obd_ops = { .o_owner = THIS_MODULE, .o_setup = lov_setup, .o_cleanup = lov_cleanup, + .o_process_config = lov_process_config, .o_connect = lov_connect, .o_disconnect = lov_disconnect, .o_statfs = lov_statfs, @@ -1863,6 +2137,7 @@ int __init lov_init(void) { struct lprocfs_static_vars lvars; int rc; + ENTRY; lprocfs_init_vars(lov, &lvars); rc = class_register_type(&lov_obd_ops, lvars.module_vars, diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index f871fff..398b462 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -621,6 +621,7 @@ int lprocfs_alloc_obd_stats(struct obd_device *obd, unsigned num_private_stats) LPROCFS_OBD_OP_INIT(num_private_stats, stats, setup); LPROCFS_OBD_OP_INIT(num_private_stats, stats, precleanup); LPROCFS_OBD_OP_INIT(num_private_stats, stats, cleanup); + LPROCFS_OBD_OP_INIT(num_private_stats, stats, process_config); LPROCFS_OBD_OP_INIT(num_private_stats, stats, postrecov); LPROCFS_OBD_OP_INIT(num_private_stats, stats, add_conn); LPROCFS_OBD_OP_INIT(num_private_stats, stats, del_conn); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index 58a0927..9277d2a 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -305,10 +305,10 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) obd->obd_force = 1; break; case 'A': - obd->obd_fail = 1; - obd->obd_no_transno = 1; LCONSOLE_WARN("Failing %s by user command\n", obd->obd_name); + obd->obd_fail = 1; + obd->obd_no_transno = 1; /* Set the obd readonly if we can */ if (OBP(obd, iocontrol)) obd_iocontrol(OBD_IOC_SET_READONLY, @@ -613,8 +613,8 @@ int class_process_config(struct lustre_cfg *lcfg) GOTO(out, err = 0); } default: { - CERROR("Unknown command: %d\n", lcfg->lcfg_command); - GOTO(out, err = -EINVAL); + err = obd_process_config(obd, sizeof(*lcfg), lcfg); + GOTO(out, err); } } diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 0629196..0e47550 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -201,7 +201,7 @@ void ptlrpc_invalidate_import(struct obd_import *imp) obd_import_event(imp->imp_obd, imp, IMP_EVENT_INVALIDATE); } -static void ptlrpc_activate_import(struct obd_import *imp) +void ptlrpc_activate_import(struct obd_import *imp) { struct obd_device *obd = imp->imp_obd; unsigned long flags; @@ -698,7 +698,7 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp) static int back_to_sleep(void *unused) { - return 0; + return 0; } int ptlrpc_disconnect_import(struct obd_import *imp) diff --git a/lustre/ptlrpc/ptlrpc_module.c b/lustre/ptlrpc/ptlrpc_module.c index 8f408d7..b4fd554 100644 --- a/lustre/ptlrpc/ptlrpc_module.c +++ b/lustre/ptlrpc/ptlrpc_module.c @@ -183,6 +183,7 @@ EXPORT_SYMBOL(ptlrpc_disconnect_import); EXPORT_SYMBOL(ptlrpc_resend); EXPORT_SYMBOL(ptlrpc_wake_delayed); EXPORT_SYMBOL(ptlrpc_set_import_active); +EXPORT_SYMBOL(ptlrpc_activate_import); EXPORT_SYMBOL(ptlrpc_deactivate_import); EXPORT_SYMBOL(ptlrpc_invalidate_import); EXPORT_SYMBOL(ptlrpc_fail_import); diff --git a/lustre/utils/Lustre/lustredb.py b/lustre/utils/Lustre/lustredb.py index 5938671..fff4be43 100644 --- a/lustre/utils/Lustre/lustredb.py +++ b/lustre/utils/Lustre/lustredb.py @@ -57,6 +57,11 @@ class LustreDB: uuids = self._get_all_refs() return uuids + def get_lov_tgts(self, tag): + """ Returns list of lov tgts. """ + tgts = self._get_lov_tgts(tag) + return tgts + def nid2server(self, nid, net_type, cluster_id): netlist = self.lookup_class('network') for net_db in netlist: @@ -182,6 +187,18 @@ class LustreDB_XML(LustreDB): for r in reflist: uuids.append(self.xml_get_ref(r)) return uuids + + def _get_lov_tgts(self, tag): + """ Get all the refs of type TAG. Returns list of lov_tgts. """ + tgts = [] + tgtlist = self.dom_node.getElementsByTagName(tag) + for tgt in tgtlist: + uuidref = tgt.getAttribute('uuidref') + index = tgt.getAttribute('index') + generation = tgt.getAttribute('generation') + active = int(tgt.getAttribute('active')) + tgts.append((uuidref, index, generation, active)) + return tgts def xmllookup_by_uuid(self, dom_node, uuid): for n in dom_node.childNodes: diff --git a/lustre/utils/lconf b/lustre/utils/lconf index fc3d656..e9ee234 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -647,22 +647,35 @@ class LCTLInterface: # create an lov def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt, - stripe_sz, stripe_off, - pattern, devlist): + stripe_sz, stripe_off, pattern): cmds = """ attach lov %s %s - lov_setup %s %d %d %d %s %s - quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, - pattern, devlist) + lov_setup %s %d %d %d %s + quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern) self.run(cmds) - # create an lov - def lov_setconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off, - pattern, devlist): + # add an OBD to a LOV + def lov_add_obd(self, name, uuid, obd_uuid, index, gen): cmds = """ - cfg_device $%s - lov_setconfig %s %d %d %d %s %s - quit""" % (mdsuuid, uuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist) + cfg_device %s + lov_modify_tgts add %s %s %s %s + quit""" % (name, name, obd_uuid, index, gen) + self.run(cmds) + + # delete an OBD from a LOV + def lov_del_obd(self, name, uuid, obd_uuid, index, gen): + cmds = """ + cfg_device %s + lov_modify_tgts del %s %s %s %s + quit""" % (name, name, obd_uuid, index, gen) + self.run(cmds) + + # deactivate an OBD + def deactivate(self, name): + cmds = """ + cfg_device %s + deactivate + quit""" % (name) self.run(cmds) # dump the log file @@ -1458,8 +1471,8 @@ class LOV(Module): self.stripe_sz = self.db.get_val_int('stripesize', 1048576) self.stripe_off = self.db.get_val_int('stripeoffset', 0) self.pattern = self.db.get_val_int('stripepattern', 0) - self.devlist = self.db.get_refs('obd') - self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist)) + self.devlist = [] + self.stripe_cnt = 0 self.osclist = [] self.desc_uuid = self.uuid self.uuid = generate_client_uuid(self.name) @@ -1468,49 +1481,74 @@ class LOV(Module): self.config_only = 1 return self.config_only = None - mds= self.db.lookup(self.mds_uuid) + mds = self.db.lookup(self.mds_uuid) self.mds_name = mds.getName() - for obd_uuid in self.devlist: + self.devlist = self.db.get_lov_tgts('lov_tgt') + for (obd_uuid, index, gen, active) in self.devlist: + if obd_uuid == '': + continue obd = self.db.lookup(obd_uuid) osc = get_osc(obd, self.uuid, fs_name) if osc: - self.osclist.append(osc) + self.osclist.append((osc, index, gen, active)) else: panic('osc not found:', obd_uuid) + if self.osclist == []: + debug("get_lov_tgts failed, using get_refs"); + index = 0 + self.devlist = self.db.get_refs('obd') + for obd_uuid in self.devlist: + obd = self.db.lookup(obd_uuid) + osc = get_osc(obd, self.uuid, fs_name) + if osc: + self.osclist.append((osc, index, 1, 1)) + else: + panic('osc not found:', obd_uuid) + index = index + 1 + if self.osclist == []: + panic('No OSCs configured for LOV') + self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist)) def prepare(self): + debug('dbg LOV prepare') if is_prepared(self.name): return - if self.config_only: - panic("Can't prepare config_only LOV ", self.name) - - for osc in self.osclist: + debug('dbg LOV prepare:', self.osclist, self.devlist) + self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz, + self.stripe_off, self.pattern, self.devlist, + self.mds_name) + lctl.lov_setup(self.name, self.uuid, + self.desc_uuid, self.mds_name, self.stripe_cnt, + self.stripe_sz, self.stripe_off, self.pattern) + if self.osclist == []: + panic('No OSCs configured for LOV?') + for (osc, index, gen, active) in self.osclist: + target_uuid = osc.target_uuid try: # Only ignore connect failures with --force, which # isn't implemented here yet. + osc.active = active osc.prepare(ignore_connect_failure=0) except CommandError, e: print "Error preparing OSC %s\n" % osc.uuid raise e - self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz, - self.stripe_off, self.pattern, self.devlist, self.mds_name) - lctl.lov_setup(self.name, self.uuid, - self.desc_uuid, self.mds_name, self.stripe_cnt, - self.stripe_sz, self.stripe_off, self.pattern, - string.join(self.devlist)) + lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen) def cleanup(self): + for (osc, index, gen, active) in self.osclist: + target_uuid = osc.target_uuid + if is_prepared(osc.name): + lctl.lov_del_obd(self.name, self.uuid, target_uuid, index, gen) + osc.cleanup() if is_prepared(self.name): Module.cleanup(self) if self.config_only: panic("Can't clean up config_only LOV ", self.name) - for osc in self.osclist: - osc.cleanup() def load_module(self): if self.config_only: panic("Can't load modules for config_only LOV ", self.name) - for osc in self.osclist: + for (osc, index, gen, active) in self.osclist: osc.load_module() break Module.load_module(self) @@ -1519,8 +1557,9 @@ class LOV(Module): if self.config_only: panic("Can't cleanup modules for config_only LOV ", self.name) Module.cleanup_module(self) - for osc in self.osclist: - osc.cleanup_module() + for (osc, index, gen, active) in self.osclist: + if active: + osc.cleanup_module() break class MDSDEV(Module): diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 30e13f1..6eb898e 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -153,12 +153,14 @@ command_t cmdlist[] = { {"detach", jt_obd_detach, 0, "remove driver (and name and uuid) from current device\n" "usage: detach"}, - {"lov_setup", jt_lcfg_lov_setup, 0, - "write setup an lov device\n" - "usage: lov_setup lov-uuid stripe-count stripe-size offset pattern UUID1 [UUID2 ...]"}, + {"lov_setup", jt_lcfg_lov_setup, 0, "create a LOV device\n" + "usage: lov_setup lov-uuid stripe-count stripe-size offset pattern"}, + {"lov_modify_tgts", jt_lcfg_lov_modify_tgts, 0, + "add or delete an obd to/from a LOV device\n" + "usage: lov_modify_tgts add|del "}, {"lov_getconfig", jt_obd_lov_getconfig, 0, "read lov configuration from an mds device\n" - "usage: lov_getconfig lov-uuid"}, + "usage: lov_getconfig /mount/path"}, {"record", jt_cfg_record, 0, "record commands that follow in log\n" "usage: record cfg-uuid-name"}, {"endrecord", jt_cfg_endrecord, 0, "stop recording\n" diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 77a3959..fa51411 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -170,16 +170,26 @@ int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count) struct lov_desc desc = { 0, }; char *buf = NULL; int max_ost_count, rc; + __u32 *obdgens; max_ost_count = (OBD_MAX_IOCTL_BUFFER - size_round(sizeof(data)) - - size_round(sizeof(desc))) / sizeof(*uuidp); + size_round(sizeof(desc))) / + (sizeof(*uuidp) + sizeof(*obdgens)); if (max_ost_count > *ost_count) max_ost_count = *ost_count; + obdgens = malloc(size_round(max_ost_count * sizeof(*obdgens))); + if (!obdgens) { + err_msg("no memory for %d generation #'s", max_ost_count); + return(-ENOMEM); + } + data.ioc_inllen1 = sizeof(desc); data.ioc_inlbuf1 = (char *)&desc; data.ioc_inllen2 = size_round(max_ost_count * sizeof(*uuidp)); data.ioc_inlbuf2 = (char *)uuidp; + data.ioc_inllen3 = size_round(max_ost_count * sizeof(*obdgens)); + data.ioc_inlbuf3 = (char *)obdgens; desc.ld_tgt_count = max_ost_count; @@ -205,6 +215,7 @@ int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count) *ost_count = desc.ld_tgt_count; out: free(buf); + free(obdgens); return rc; } diff --git a/lustre/utils/lustre_cfg.c b/lustre/utils/lustre_cfg.c index cb0f987f..05abe53 100644 --- a/lustre/utils/lustre_cfg.c +++ b/lustre/utils/lustre_cfg.c @@ -57,30 +57,30 @@ static char * lcfg_devname; void lcfg_set_devname(char *name) { - if (lcfg_devname) - free(lcfg_devname); - lcfg_devname = strdup(name); + if (lcfg_devname) + free(lcfg_devname); + lcfg_devname = strdup(name); } int jt_lcfg_device(int argc, char **argv) { - char *name; + char *name; if (argc == 1) { - printf("current device is %s\n", lcfg_devname? : "not set"); - return 0; - } else if (argc != 2) { + printf("current device is %s\n", lcfg_devname? : "not set"); + return 0; + } else if (argc != 2) { return CMD_HELP; - } + } - name = argv[1]; + name = argv[1]; - /* quietly strip the unnecessary '$' */ - if (*name == '$') - name++; + /* quietly strip the unnecessary '$' */ + if (*name == '$') + name++; - lcfg_set_devname(name); + lcfg_set_devname(name); return 0; } @@ -108,8 +108,8 @@ int jt_lcfg_attach(int argc, char **argv) } else { fprintf(stderr, "error: %s: LCFG_ATTACH requires a name\n", jt_cmdname(argv[0])); - return -EINVAL; - } + return -EINVAL; + } if (argc == 4) { lustre_cfg_bufs_set_string(&bufs, 2, argv[3]); @@ -124,7 +124,7 @@ int jt_lcfg_attach(int argc, char **argv) } else if (argc == 3) { char name[1024]; - lcfg_set_devname(argv[2]); + lcfg_set_devname(argv[2]); if (strlen(argv[2]) > 128) { printf("Name too long to set environment\n"); return -EINVAL; @@ -135,8 +135,8 @@ int jt_lcfg_attach(int argc, char **argv) printf("error setting env variable %s\n", name); } } else { - lcfg_set_devname(argv[2]); - } + lcfg_set_devname(argv[2]); + } return rc; } @@ -152,7 +152,7 @@ int jt_lcfg_setup(int argc, char **argv) fprintf(stderr, "%s: please use 'cfg_device name' to set the " "device name for config commands.\n", jt_cmdname(argv[0])); - return -EINVAL; + return -EINVAL; } lustre_cfg_bufs_reset(&bufs, lcfg_devname); @@ -184,7 +184,7 @@ int jt_obd_detach(int argc, char **argv) fprintf(stderr, "%s: please use 'cfg_device name' to set the " "device name for config commands.\n", jt_cmdname(argv[0])); - return -EINVAL; + return -EINVAL; } lustre_cfg_bufs_reset(&bufs, lcfg_devname); @@ -216,7 +216,7 @@ int jt_obd_cleanup(int argc, char **argv) fprintf(stderr, "%s: please use 'cfg_device name' to set the " "device name for config commands.\n", jt_cmdname(argv[0])); - return -EINVAL; + return -EINVAL; } lustre_cfg_bufs_reset(&bufs, lcfg_devname); @@ -344,11 +344,13 @@ int jt_lcfg_lov_setup(int argc, char **argv) struct lustre_cfg_bufs bufs; struct lustre_cfg *lcfg; struct lov_desc desc; - struct obd_uuid *uuidarray = NULL, *ptr; - int rc, i; + int rc; char *end; - if (argc <= 6) + /* argv: lov_setup + * [ ] + */ + if (argc < 6 || argc > 7) return CMD_HELP; if (strlen(argv[1]) > sizeof(desc.ld_uuid) - 1) { @@ -360,7 +362,7 @@ int jt_lcfg_lov_setup(int argc, char **argv) memset(&desc, 0, sizeof(desc)); obd_str2uuid(&desc.ld_uuid, argv[1]); - desc.ld_tgt_count = argc - 6; + desc.ld_tgt_count = 0; desc.ld_magic = LOV_DESC_MAGIC; desc.ld_default_stripe_count = strtoul(argv[2], &end, 0); if (*end) { @@ -368,13 +370,6 @@ int jt_lcfg_lov_setup(int argc, char **argv) jt_cmdname(argv[0]), argv[2]); return CMD_HELP; } - if (desc.ld_default_stripe_count > desc.ld_tgt_count) { - fprintf(stderr, - "error: %s: default stripe count %u > OST count %u\n", - jt_cmdname(argv[0]), desc.ld_default_stripe_count, - desc.ld_tgt_count); - return -EINVAL; - } desc.ld_default_stripe_size = strtoull(argv[3], &end, 0); if (*end) { @@ -407,33 +402,25 @@ int jt_lcfg_lov_setup(int argc, char **argv) return CMD_HELP; } - /* NOTE: it is possible to overwrite the default striping parameters, - * but EXTREME care must be taken when saving the OST UUID list. - * It must be EXACTLY the same, or have only additions at the - * end of the list, or only overwrite individual OST entries - * that are restored from backups of the previous OST. - */ - uuidarray = calloc(desc.ld_tgt_count, sizeof(*uuidarray)); - if (!uuidarray) { - fprintf(stderr, "error: %s: no memory for %d UUIDs\n", - jt_cmdname(argv[0]), desc.ld_tgt_count); - rc = -ENOMEM; - goto out; - } - for (i = 6, ptr = uuidarray; i < argc; i++, ptr++) { - if (strlen(argv[i]) >= sizeof(*ptr)) { - fprintf(stderr, "error: %s: arg %d (%s) too long\n", - jt_cmdname(argv[0]), i, argv[i]); - rc = -EINVAL; - goto out; + if (argc == 7) { + desc.ld_tgt_count = strtoul(argv[6], &end, 0); + if (*end) { + fprintf(stderr, "error: %s: bad target count '%s'\n", + jt_cmdname(argv[0]), argv[6]); + return CMD_HELP; + } + if (desc.ld_default_stripe_count > desc.ld_tgt_count) { + fprintf(stderr, + "error: %s: default stripe count %u > " + "OST count %u\n", jt_cmdname(argv[0]), + desc.ld_default_stripe_count, + desc.ld_tgt_count); + return -EINVAL; } - strcpy((char *)ptr, argv[i]); } lustre_cfg_bufs_reset(&bufs, lcfg_devname); lustre_cfg_bufs_set(&bufs, 1, &desc, sizeof(desc)); - lustre_cfg_bufs_set(&bufs, 2, uuidarray, - desc.ld_tgt_count * sizeof(*uuidarray)); lcfg = lustre_cfg_new(LCFG_SETUP, &bufs); rc = lcfg_ioctl(argv[0], OBD_DEV_ID, lcfg); @@ -441,11 +428,72 @@ int jt_lcfg_lov_setup(int argc, char **argv) if (rc) fprintf(stderr, "error: %s: ioctl error: %s\n", jt_cmdname(argv[0]), strerror(rc = errno)); -out: - free(uuidarray); return rc; } +int jt_lcfg_lov_modify_tgts(int argc, char **argv) +{ + struct lustre_cfg_bufs bufs; + struct lustre_cfg *lcfg; + char *end; + int cmd = 0; + int index; + int gen; + int rc; + + /* argv: lov_modify_tgts */ + if (argc != 6) + return CMD_HELP; + + if (!strncmp(argv[1], "add", 4)) { + cmd = LCFG_LOV_ADD_OBD; + } else if (!strncmp(argv[1], "del", 4)) { + cmd = LCFG_LOV_DEL_OBD; + } else { + fprintf(stderr, "error: %s: bad operation '%s'\n", + jt_cmdname(argv[0]), argv[1]); + return CMD_HELP; + } + + lustre_cfg_bufs_reset(&bufs, lcfg_devname); + + if (((index = strlen(argv[3]) + 1)) > sizeof(struct obd_uuid)) { + fprintf(stderr, + "error: %s: OBD uuid '%s' longer than "LPSZ" chars\n", + jt_cmdname(argv[0]), argv[3], + sizeof(struct obd_uuid) - 1); + return -EINVAL; + } + lustre_cfg_bufs_set(&bufs, 1, argv[3], index); + + index = strtoul(argv[4], &end, 0); + if (*end) { + fprintf(stderr, "error: %s: bad OBD index '%s'\n", + jt_cmdname(argv[0]), argv[4]); + return CMD_HELP; + } + lustre_cfg_bufs_set(&bufs, 2, argv[4], strlen(argv[4])); + + gen = strtoul(argv[5], &end, 0); + if (*end) { + fprintf(stderr, "error: %s: bad OBD generation '%s'\n", + jt_cmdname(argv[0]), argv[5]); + return CMD_HELP; + } + lustre_cfg_bufs_set(&bufs, 3, argv[5], strlen(argv[5])); + + lcfg = lustre_cfg_new(cmd, &bufs); + rc = lcfg_ioctl(argv[0], OBD_DEV_ID, lcfg); + lustre_cfg_free(lcfg); + if (rc) + fprintf(stderr, "error: %s: ioctl error: %s\n", + jt_cmdname(argv[0]), strerror(rc = errno)); + + return rc; +} + + + int jt_lcfg_mount_option(int argc, char **argv) { int rc; diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index 2190928..a1048cc 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -1590,6 +1590,7 @@ int jt_obd_lov_getconfig(int argc, char **argv) struct obd_ioctl_data data; struct lov_desc desc; struct obd_uuid *uuidarray; + __u32 *obdgens; char *path; int rc, fd; @@ -1603,13 +1604,14 @@ int jt_obd_lov_getconfig(int argc, char **argv) if (fd < 0) { fprintf(stderr, "open \"%s\" failed: %s\n", path, strerror(errno)); - return -1; + return -errno; } memset(&desc, 0, sizeof(desc)); obd_str2uuid(&desc.ld_uuid, argv[1]); desc.ld_tgt_count = ((OBD_MAX_IOCTL_BUFFER-sizeof(data)-sizeof(desc)) / - sizeof(*uuidarray)); + (sizeof(*uuidarray) + sizeof(*obdgens))); + repeat: uuidarray = calloc(desc.ld_tgt_count, sizeof(*uuidarray)); @@ -1619,27 +1621,38 @@ repeat: rc = -ENOMEM; goto out; } + obdgens = calloc(desc.ld_tgt_count, sizeof(*obdgens)); + if (!obdgens) { + fprintf(stderr, "error: %s: no memory for %d generation #'s\n", + jt_cmdname(argv[0]), desc.ld_tgt_count); + rc = -ENOMEM; + goto out_uuidarray; + } data.ioc_inllen1 = sizeof(desc); data.ioc_inlbuf1 = (char *)&desc; data.ioc_inllen2 = desc.ld_tgt_count * sizeof(*uuidarray); data.ioc_inlbuf2 = (char *)uuidarray; + data.ioc_inllen3 = desc.ld_tgt_count * sizeof(*obdgens); + data.ioc_inlbuf3 = (char *)obdgens; if (obd_ioctl_pack(&data, &buf, max)) { fprintf(stderr, "error: %s: invalid ioctl\n", jt_cmdname(argv[0])); rc = -EINVAL; - goto out; + goto out_obdgens; } rc = ioctl(fd, OBD_IOC_LOV_GET_CONFIG, buf); if (rc == -ENOSPC) { free(uuidarray); + free(obdgens); goto repeat; } else if (rc) { fprintf(stderr, "error: %s: ioctl error: %s\n", jt_cmdname(argv[0]), strerror(rc = errno)); } else { - struct obd_uuid *ptr; + struct obd_uuid *uuidp; + __u32 *genp; int i; if (obd_ioctl_unpack(&data, buf, max)) { @@ -1656,11 +1669,17 @@ repeat: desc.ld_default_stripe_offset); printf("default_stripe_pattern: %u\n", desc.ld_pattern); printf("obd_count: %u\n", desc.ld_tgt_count); - for (i = 0, ptr = uuidarray; i < desc.ld_tgt_count; i++, ptr++) - printf("%u: %s\n", i, (char *)ptr); - } -out: + printf("OBDS:\tobdidx\t\tobdgen\t\t obduuid\n"); + uuidp = uuidarray; + genp = obdgens; + for (i = 0; i < desc.ld_tgt_count; i++, uuidp++, genp++) + printf("\t%6u\t%14u\t\t %s\n", i, *genp, (char *)uuidp); + } +out_obdgens: + free(obdgens); +out_uuidarray: free(uuidarray); +out: close(fd); return rc; } diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index e1c28ba..fed08df 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -81,6 +81,7 @@ int jt_lcfg_setup(int argc, char **argv); int jt_lcfg_add_uuid(int argc, char **argv); int jt_lcfg_del_uuid(int argc, char **argv); int jt_lcfg_lov_setup(int argc, char **argv); +int jt_lcfg_lov_modify_tgts(int argc, char **argv); int jt_lcfg_mount_option(int argc, char **argv); int jt_lcfg_del_mount_option(int argc, char **argv); int jt_lcfg_set_timeout(int argc, char **argv); -- 1.8.3.1