From: James Simmons Date: Wed, 5 Mar 2025 19:50:53 +0000 (-0500) Subject: LU-11850 lov: migrate completely to lu_tgt_descs API X-Git-Tag: 2.16.53~23 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=f70332330acaff7d52a21700726e7f89a85789b2;p=fs%2Flustre-release.git LU-11850 lov: migrate completely to lu_tgt_descs API The lov target handling was written before the generic lu_tgt was written. Migrate to this newer API so lov can be treated the same like lmv and lod. With the changes we have the new lov_foreach_tgt() macro that tranverses all the registered targets of total amount ltd->ltd_tgts_size. Also lov_tgt() was created to extract a target by its index. Internally a bitmap is used to tell if the tgt has been setup and ltd->ltd_tgts_size defines the largest possible index. Another change is that since that largest OST offset that a striped lustre file can have is 65503 we reduce the largest index possible for an OST since the last OSTs could never be used. Fixes: 1a6ef725c2 ("LU-16938 utils: setstripe overstripe multiple OST count") Change-Id: If3f53b2a4589f93a024fa026ba377e2175282c29 Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51959 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Feng Lei Reviewed-by: Lai Siyao Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index b4ddc4c..fb984d1 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -1603,7 +1603,7 @@ struct lu_tgt_descs { /* TGTs scheduled to be deleted */ __u32 ltd_death_row; /* Table refcount used for delayed deletion */ - int ltd_refcount; + atomic_t ltd_refcount; /* mutex to serialize concurrent updates to the tgt table */ struct mutex ltd_mutex; /* read/write semaphore used for array relocation */ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index aebcb36..2e8c184 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -393,15 +393,8 @@ struct lov_md_tgt_desc { }; struct lov_obd { - struct lov_desc desc; - struct lov_tgt_desc **lov_tgts; /* sparse array */ - struct lu_tgt_pool lov_packed; /* all OSTs in a packed - * array */ - struct mutex lov_lock; + struct lu_tgt_descs lov_ost_descs; struct obd_connect_data lov_ocd; - atomic_t lov_refcount; - __u32 lov_death_row; /* tgts scheduled to be deleted */ - __u32 lov_tgt_size; /* size of tgts array */ int lov_connects; int lov_pool_count; struct rhashtable lov_pools_hash_body; /* used for key access */ diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 52af57a..e4474d5 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -36,7 +36,7 @@ void lod_getref(struct lod_tgt_descs *ltd) { down_read(<d->ltd_rw_sem); mutex_lock(<d->ltd_mutex); - ltd->ltd_refcount++; + atomic_inc(<d->ltd_refcount); mutex_unlock(<d->ltd_mutex); } @@ -53,8 +53,7 @@ void lod_getref(struct lod_tgt_descs *ltd) void lod_putref(struct lod_device *lod, struct lod_tgt_descs *ltd) { mutex_lock(<d->ltd_mutex); - ltd->ltd_refcount--; - if (ltd->ltd_refcount == 0 && ltd->ltd_death_row) { + if (atomic_dec_and_test(<d->ltd_refcount) && ltd->ltd_death_row) { struct lod_tgt_desc *tgt_desc, *tmp; LIST_HEAD(kill); diff --git a/lustre/lov/lov_cl_internal.h b/lustre/lov/lov_cl_internal.h index a9845d4..3f75ca8 100644 --- a/lustre/lov/lov_cl_internal.h +++ b/lustre/lov/lov_cl_internal.h @@ -578,9 +578,6 @@ struct lu_object *lovsub_object_alloc(const struct lu_env *env, int lov_io_layout_at(struct lov_io *lio, __u64 offset); bool lov_io_layout_at_confirm(struct lov_io *lio, int entry, __u64 offset); -#define lov_foreach_target(lov, var) \ - for (var = 0; var < lov_targets_nr(lov); ++var) - static inline struct lu_extent *lov_io_extent(struct lov_io *io, int i) { return &lov_lse(io->lis_object, i)->lsme_extent; @@ -715,11 +712,6 @@ static inline struct lov_io *cl2lov_io(const struct lu_env *env, return lio; } -static inline int lov_targets_nr(const struct lov_device *lov) -{ - return lov->ld_lov->desc.ld_tgt_count; -} - static inline struct lov_thread_info *lov_env_info(const struct lu_env *env) { struct lov_thread_info *info; diff --git a/lustre/lov/lov_dev.c b/lustre/lov/lov_dev.c index d24c156..0352ea3 100644 --- a/lustre/lov/lov_dev.c +++ b/lustre/lov/lov_dev.c @@ -190,16 +190,16 @@ static struct lu_device *lov_device_fini(const struct lu_env *env, struct lu_device *d) { struct lov_device *ld = lu2lov_dev(d); - int i; - - LASSERT(ld->ld_lov != NULL); + LASSERT(ld->ld_lov); if (ld->ld_lmv) { class_decref(ld->ld_lmv, "lov", d); ld->ld_lmv = NULL; } if (ld->ld_md_tgts) { + int i; + for (i = 0; i < ld->ld_md_tgts_nr; i++) { if (!ld->ld_md_tgts[i].ldm_mdc) continue; @@ -211,13 +211,15 @@ static struct lu_device *lov_device_fini(const struct lu_env *env, } if (ld->ld_target) { - lov_foreach_target(ld, i) { + struct lov_tgt_desc *desc; + + lov_foreach_tgt(ld->ld_lov, desc) { struct lovsub_device *lsd; - lsd = ld->ld_target[i]; + lsd = ld->ld_target[desc->ltd_index]; if (lsd) { cl_stack_fini(env, lovsub2cl_dev(lsd)); - ld->ld_target[i] = NULL; + ld->ld_target[desc->ltd_index] = NULL; } } } @@ -239,8 +241,8 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d, const char *name, struct lu_device *next) { struct lov_device *ld = lu2lov_dev(d); - int i; - int rc = 0; + struct lov_tgt_desc *desc; + int rc = 0, i; /* check all added already MDC subdevices and initialize them */ for (i = 0; i < ld->ld_md_tgts_nr; i++) { @@ -265,14 +267,9 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d, if (!ld->ld_target) RETURN(0); - lov_foreach_target(ld, i) { + lov_foreach_tgt(ld->ld_lov, desc) { struct lovsub_device *lsd; struct cl_device *cl; - struct lov_tgt_desc *desc; - - desc = ld->ld_lov->lov_tgts[i]; - if (!desc) - continue; cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type, desc->ltd_obd->obd_lu_dev); @@ -280,7 +277,7 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d, GOTO(out_err, rc = PTR_ERR(cl)); lsd = cl2lovsub_dev(cl); - ld->ld_target[i] = lsd; + ld->ld_target[desc->ltd_index] = lsd; } ld->ld_flags |= LOV_DEV_INITIALIZED; RETURN(0); @@ -355,7 +352,7 @@ static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev) ENTRY; result = 0; - tgt_size = dev->ld_lov->lov_tgt_size; + tgt_size = dev->ld_lov->lov_ost_descs.ltd_tgts_size; sub_size = dev->ld_target_nr; if (sub_size < tgt_size) { struct lovsub_device **newd; @@ -399,12 +396,10 @@ static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev, int rc; ENTRY; - lov_tgts_getref(obd); - - tgt = obd->u.lov.lov_tgts[index]; - LASSERT(tgt != NULL); - LASSERT(tgt->ltd_obd != NULL); + tgt = lov_tgt(&obd->u.lov, index); + LASSERT(tgt); + LASSERT(tgt->ltd_obd); if (!test_bit(OBDF_SET_UP, tgt->ltd_obd->obd_flags)) { CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid)); diff --git a/lustre/lov/lov_ea.c b/lustre/lov/lov_ea.c index 8f55172..efc6fcc 100644 --- a/lustre/lov/lov_ea.c +++ b/lustre/lov/lov_ea.c @@ -172,6 +172,7 @@ lsme_unpack(struct lov_obd *lov, struct lov_mds_md *lmm, size_t buf_size, const char *pool_name, bool inited, struct lov_ost_data_v1 *objects, loff_t *maxbytes) { + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; struct lov_stripe_md_entry *lsme; size_t lsme_size; loff_t min_stripe_maxbytes = 0; @@ -241,7 +242,7 @@ lsme_unpack(struct lov_obd *lov, struct lov_mds_md *lmm, size_t buf_size, for (i = 0; i < stripe_count; i++) { struct lov_oinfo *loi; - struct lov_tgt_desc *ltd = NULL; + struct lov_tgt_desc *tgt = NULL; static time64_t next_print; unsigned int level; @@ -258,8 +259,8 @@ lsme_unpack(struct lov_obd *lov, struct lov_mds_md *lmm, size_t buf_size, continue; retry_new_ost: - if (unlikely((u32)loi->loi_ost_idx >= lov->desc.ld_tgt_count || - !(ltd = lov->lov_tgts[loi->loi_ost_idx]))) { + if (unlikely((u32)loi->loi_ost_idx >= ltd->ltd_tgts_size || + !(tgt = lov_tgt(lov, loi->loi_ost_idx)))) { time64_t now = ktime_get_seconds(); /* print message on the first hit, error if giving up */ @@ -274,18 +275,19 @@ retry_new_ost: /* log debug every loop, just to see it is trying */ CDEBUG_LIMIT(level, - (u32)loi->loi_ost_idx < lov->desc.ld_tgt_count ? - "%s: FID "DOSTID" OST index %d/%u missing\n" : - "%s: FID "DOSTID" OST index %d more than OST count %u\n", - lov->desc.ld_uuid.uuid, POSTID(&loi->loi_oi), - loi->loi_ost_idx, lov->desc.ld_tgt_count); + (u32)loi->loi_ost_idx < ltd->ltd_tgts_size ? + "%s: FID "DOSTID" OST index %d/%u missing\n" : + "%s: FID "DOSTID" OST index %d more than OST count %u\n", + ltd->ltd_lov_desc.ld_uuid.uuid, + POSTID(&loi->loi_oi), loi->loi_ost_idx, + ltd->ltd_tgts_size); if ((u32)loi->loi_ost_idx >= LOV_V1_INSANE_STRIPE_INDEX) GOTO(out_lsme, rc = -EINVAL); if (now > next_print) { LCONSOLE_INFO("%s: wait %ds while client connects to new OST\n", - lov->desc.ld_uuid.uuid, + ltd->ltd_lov_desc.ld_uuid.uuid, (int)(retry_limit - now)); next_print = retry_limit + 600; } @@ -298,7 +300,7 @@ retry_new_ost: GOTO(out_lsme, rc = -EINVAL); } - lov_bytes = lov_tgt_maxbytes(ltd); + lov_bytes = lov_tgt_maxbytes(tgt); if (min_stripe_maxbytes == 0 || lov_bytes < min_stripe_maxbytes) min_stripe_maxbytes = lov_bytes; } @@ -309,7 +311,7 @@ retry_new_ost: if (stripe_count == 0) stripe_count = lsme->lsme_stripe_count <= 0 ? - lov->desc.ld_tgt_count : + ltd->ltd_lov_desc.ld_tgt_count : lsme->lsme_stripe_count; if (min_stripe_maxbytes <= LLONG_MAX / stripe_count) { diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h index 0de5e21..867df6b 100644 --- a/lustre/lov/lov_internal.h +++ b/lustre/lov/lov_internal.h @@ -241,8 +241,17 @@ extern struct kmem_cache *lov_oinfo_slab; extern struct lu_kmem_descr lov_caches[]; +static inline struct lu_tgt_desc *lov_tgt(struct lov_obd *lov, u32 index) +{ + return index < lov->lov_ost_descs.ltd_tgts_size ? + LTD_TGT(&lov->lov_ost_descs, index) : NULL; +} + #define lov_uuid2str(lv, index) \ - (char *)((lv)->lov_tgts[index]->ltd_uuid.uuid) + (char *)(lov_tgt(lv, index)->ltd_uuid.uuid) + +#define lov_foreach_tgt(lov, tgt) \ + ltd_foreach_tgt(&(lov)->lov_ost_descs, tgt) /* lov_merge.c */ int lov_merge_lvb_kms(struct lov_stripe_md *lsm, int index, diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index f4c8584..d03f886 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -36,46 +36,44 @@ #include "lov_internal.h" /* Keep a refcount of lov->tgt usage to prevent racing with addition/deletion. - * Any function that expects lov_tgts to remain stationary must take a ref. */ + * Any function that expects lov_tgts to remain stationary must take a ref. + */ void lov_tgts_getref(struct obd_device *obd) { - struct lov_obd *lov = &obd->u.lov; + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; /* nobody gets through here until lov_putref is done */ - mutex_lock(&lov->lov_lock); - atomic_inc(&lov->lov_refcount); - mutex_unlock(&lov->lov_lock); + mutex_lock(<d->ltd_mutex); + atomic_inc(<d->ltd_refcount); + mutex_unlock(<d->ltd_mutex); } static void __lov_del_obd(struct obd_device *obd, struct lov_tgt_desc *tgt); void lov_tgts_putref(struct obd_device *obd) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_obd *lov = &obd->u.lov; - mutex_lock(&lov->lov_lock); + mutex_lock(<d->ltd_mutex); /* ok to dec to 0 more than once -- ltd_exp's will be null */ - if (atomic_dec_and_test(&lov->lov_refcount) && lov->lov_death_row) { - LIST_HEAD(kill); + if (atomic_dec_and_test(<d->ltd_refcount) && ltd->ltd_death_row) { struct lov_tgt_desc *tgt, *n; - int i; + LIST_HEAD(kill); CDEBUG(D_CONFIG, "destroying %d lov targets\n", - lov->lov_death_row); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - tgt = lov->lov_tgts[i]; - - if (!tgt || !tgt->ltd_reap) + ltd->ltd_death_row); + lov_foreach_tgt(lov, tgt) { + if (!tgt->ltd_reap) continue; + list_add(&tgt->ltd_kill, &kill); - /* XXX - right now there is a dependency on ld_tgt_count - * being the maximum tgt index for computing the - * mds_max_easize. So we can't shrink it. */ - lu_tgt_pool_remove(&lov->lov_packed, i); - lov->lov_tgts[i] = NULL; - lov->lov_death_row--; + lu_tgt_pool_remove(<d->ltd_tgt_pool, + tgt->ltd_index); + ltd_del_tgt(ltd, tgt); + ltd->ltd_death_row--; } - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); list_for_each_entry_safe(tgt, n, &kill, ltd_kill) { list_del(&tgt->ltd_kill); @@ -83,7 +81,7 @@ void lov_tgts_putref(struct obd_device *obd) __lov_del_obd(obd, tgt); } } else { - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); } } @@ -98,14 +96,16 @@ static int lov_connect_osc(struct obd_device *obd, u32 index, int activate, struct obd_device *tgt_obd; static struct obd_uuid lov_osc_uuid = { "LOV_OSC_UUID" }; struct obd_import *imp; + struct lov_tgt_desc *tgt; int rc; - ENTRY; - if (lov->lov_tgts[index] == NULL) + ENTRY; + tgt = lov_tgt(lov, index); + if (!tgt) RETURN(-EINVAL); - tgt_uuid = &lov->lov_tgts[index]->ltd_uuid; - tgt_obd = lov->lov_tgts[index]->ltd_obd; + tgt_uuid = &tgt->ltd_uuid; + tgt_obd = tgt->ltd_obd; if (!test_bit(OBDF_SET_UP, tgt_obd->obd_flags)) { rc = -EINVAL; @@ -145,16 +145,16 @@ static int lov_connect_osc(struct obd_device *obd, u32 index, int activate, RETURN(0); } - rc = obd_connect(NULL, &lov->lov_tgts[index]->ltd_exp, tgt_obd, - &lov_osc_uuid, data, lov->lov_cache); - if (rc || !lov->lov_tgts[index]->ltd_exp) { + rc = obd_connect(NULL, &tgt->ltd_exp, tgt_obd, &lov_osc_uuid, data, + lov->lov_cache); + if (rc || !tgt->ltd_exp) { CERROR("%s: target connect error: rc = %d\n", obd_uuid2str(tgt_uuid), rc); obd_register_observer(tgt_obd, NULL); RETURN(-ENODEV); } - lov->lov_tgts[index]->ltd_reap = 0; + tgt->ltd_reap = 0; CDEBUG(D_CONFIG, "Connected tgt idx %d %s (%s) %sactive\n", index, obd_uuid2str(tgt_uuid), tgt_obd->obd_name, activate ? "":"in"); @@ -167,8 +167,7 @@ static int lov_connect_osc(struct obd_device *obd, u32 index, int activate, if (rc) { CERROR("%s: can't register LOV target /sys/fs/lustre/%s/%s/target_obds/%s : rc = %d\n", obd->obd_name, obd->obd_type->typ_name, - obd->obd_name, - lov->lov_tgts[index]->ltd_exp->exp_obd->obd_name, + obd->obd_name, tgt->ltd_exp->exp_obd->obd_name, rc); } } @@ -182,9 +181,9 @@ static int lov_connect(const struct lu_env *env, struct obd_export **exp, struct lov_obd *lov = &obd->u.lov; struct lov_tgt_desc *tgt; struct lustre_handle conn; - int i, rc; - ENTRY; + int rc; + ENTRY; CDEBUG(D_CONFIG, "connect #%d\n", lov->lov_connects); rc = class_connect(&conn, obd, cluuid); @@ -208,25 +207,26 @@ static int lov_connect(const struct lu_env *env, struct obd_export **exp, cl_cache_incref(lov->lov_cache); } - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - tgt = lov->lov_tgts[i]; - if (!tgt || obd_uuid_empty(&tgt->ltd_uuid)) + lov_foreach_tgt(lov, tgt) { + if (obd_uuid_empty(&tgt->ltd_uuid)) continue; + /* Flags will be lowest common denominator */ - rc = lov_connect_osc(obd, i, tgt->ltd_activate, &lov->lov_ocd); + rc = lov_connect_osc(obd, tgt->ltd_index, tgt->ltd_activate, + &lov->lov_ocd); if (rc) { CERROR("%s: lov connect tgt %d failed: rc = %d\n", - obd->obd_name, i, rc); + obd->obd_name, tgt->ltd_index, rc); continue; } /* connect to administrative disabled ost */ - if (!lov->lov_tgts[i]->ltd_exp) + if (!tgt->ltd_exp) continue; - rc = lov_notify(obd, lov->lov_tgts[i]->ltd_exp->exp_obd, + rc = lov_notify(obd, tgt->ltd_exp->exp_obd, OBD_NOTIFY_CONNECT); if (rc) { - CERROR("%s error sending notify: rc = %d\n", + CERROR("%s: error sending notify: rc = %d\n", obd->obd_name, rc); } } @@ -238,18 +238,19 @@ static int lov_connect(const struct lu_env *env, struct obd_export **exp, static int lov_disconnect_obd(struct obd_device *obd, struct lov_tgt_desc *tgt) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_obd *lov = &obd->u.lov; struct obd_device *osc_obd; int rc; - ENTRY; + ENTRY; osc_obd = class_exp2obd(tgt->ltd_exp); CDEBUG(D_CONFIG, "%s: disconnecting target %s\n", obd->obd_name, osc_obd ? osc_obd->obd_name : ""); if (tgt->ltd_active) { tgt->ltd_active = 0; - lov->desc.ld_active_tgt_count--; + ltd->ltd_lov_desc.ld_active_tgt_count--; tgt->ltd_exp->exp_obd->obd_inactive = 1; } @@ -284,11 +285,11 @@ static int lov_disconnect(struct obd_export *exp) { struct obd_device *obd = class_exp2obd(exp); struct lov_obd *lov = &obd->u.lov; - u32 index; + struct lu_tgt_desc *tgt; int rc; ENTRY; - if (!lov->lov_tgts) + if (!lov->lov_ost_descs.ltd_tgts_size) goto out; /* Only disconnect the underlying layers on the final disconnect. */ @@ -308,12 +309,10 @@ static int lov_disconnect(struct obd_export *exp) /* hold another ref so lov_del_obd() doesn't spin in putref each time */ lov_tgts_getref(obd); - - for (index = 0; index < lov->desc.ld_tgt_count; index++) { - if (lov->lov_tgts[index] && lov->lov_tgts[index]->ltd_exp) { + lov_foreach_tgt(lov, tgt) { + if (tgt->ltd_exp) { /* Disconnection is the last we know about an OBD */ - lov_del_target(obd, index, NULL, - lov->lov_tgts[index]->ltd_gen); + lov_del_target(obd, tgt->ltd_index, NULL, tgt->ltd_gen); } } lov_tgts_putref(obd); @@ -333,24 +332,25 @@ out: static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid, enum obd_notify_event ev) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_obd *lov = &obd->u.lov; struct lov_tgt_desc *tgt; - int index; bool activate, active; - ENTRY; + int index = -EINVAL; + ENTRY; CDEBUG(D_INFO, "Searching in lov %p for uuid %s event(%d)\n", lov, uuid->uuid, ev); lov_tgts_getref(obd); - for (index = 0; index < lov->desc.ld_tgt_count; index++) { - tgt = lov->lov_tgts[index]; - if (tgt && obd_uuid_equals(uuid, &tgt->ltd_uuid)) + lov_foreach_tgt(lov, tgt) { + if (obd_uuid_equals(uuid, &tgt->ltd_uuid)) { + index = tgt->ltd_index; break; + } } - - if (index == lov->desc.ld_tgt_count) - GOTO(out, index = -EINVAL); + if (index < 0) + GOTO(out, index); if (ev == OBD_NOTIFY_DEACTIVATE || ev == OBD_NOTIFY_ACTIVATE) { activate = (ev == OBD_NOTIFY_ACTIVATE); @@ -360,8 +360,8 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid, * we make up for it here. */ if (activate && !tgt->ltd_exp) { - int rc; struct obd_uuid lov_osc_uuid = {"LOV_OSC_UUID"}; + int rc; rc = obd_connect(NULL, &tgt->ltd_exp, tgt->ltd_obd, &lov_osc_uuid, &lov->lov_ocd, @@ -370,18 +370,18 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid, GOTO(out, index = rc); } - if (lov->lov_tgts[index]->ltd_activate == activate) { + if (tgt->ltd_activate == activate) { CDEBUG(D_INFO, "OSC %s already %sactivate!\n", uuid->uuid, activate ? "" : "de"); } else { - lov->lov_tgts[index]->ltd_activate = activate; + tgt->ltd_activate = activate; CDEBUG(D_CONFIG, "%sactivate OSC %s\n", activate ? "" : "de", obd_uuid2str(uuid)); } } else if (ev == OBD_NOTIFY_INACTIVE || ev == OBD_NOTIFY_ACTIVE) { active = (ev == OBD_NOTIFY_ACTIVE); - if (lov->lov_tgts[index]->ltd_active == active) { + if (tgt->ltd_active == active) { CDEBUG(D_INFO, "OSC %s already %sactive!\n", uuid->uuid, active ? "" : "in"); GOTO(out, index); @@ -389,13 +389,13 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid, CDEBUG(D_CONFIG, "Marking OSC %s %sactive\n", obd_uuid2str(uuid), active ? "" : "in"); - lov->lov_tgts[index]->ltd_active = active; + tgt->ltd_active = active; if (active) { - lov->desc.ld_active_tgt_count++; - lov->lov_tgts[index]->ltd_exp->exp_obd->obd_inactive = 0; + ltd->ltd_lov_desc.ld_active_tgt_count++; + tgt->ltd_exp->exp_obd->obd_inactive = 0; } else { - lov->desc.ld_active_tgt_count--; - lov->lov_tgts[index]->ltd_exp->exp_obd->obd_inactive = 1; + ltd->ltd_lov_desc.ld_active_tgt_count--; + tgt->ltd_exp->exp_obd->obd_inactive = 1; } } else { CERROR("%s: unknown event %d for uuid %s\n", obd->obd_name, @@ -404,9 +404,9 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid, if (tgt->ltd_exp) CDEBUG(D_INFO, "%s: lov idx %d conn %llx\n", obd_uuid2str(uuid), - index, tgt->ltd_exp->exp_handle.h_cookie); + tgt->ltd_index, tgt->ltd_exp->exp_handle.h_cookie); - out: +out: lov_tgts_putref(obd); RETURN(index); } @@ -460,6 +460,7 @@ static int lov_add_target(struct obd_device *obd, struct obd_uuid *uuidp, u32 index, int gen, int active) { struct lov_obd *lov = &obd->u.lov; + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; struct lov_tgt_desc *tgt; struct obd_device *tgt_obd; int rc; @@ -478,80 +479,52 @@ static int lov_add_target(struct obd_device *obd, struct obd_uuid *uuidp, if (tgt_obd == NULL) RETURN(-EINVAL); - mutex_lock(&lov->lov_lock); - - if ((index < lov->lov_tgt_size) && (lov->lov_tgts[index] != NULL)) { - tgt = lov->lov_tgts[index]; + mutex_lock(<d->ltd_mutex); + /* check that the index is allocated in the bitmap */ + if (index < ltd->ltd_tgts_size && + test_bit(index, ltd->ltd_tgt_bitmap) && lov_tgt(lov, index)) { rc = -EEXIST; CERROR("%s: UUID %s already assigned at index %d: rc = %d\n", - obd->obd_name, obd_uuid2str(&tgt->ltd_uuid), index, rc); - mutex_unlock(&lov->lov_lock); + obd->obd_name, obd_uuid2str(uuidp), index, rc); + mutex_unlock(<d->ltd_mutex); RETURN(rc); } - if (index >= lov->lov_tgt_size) { - /* We need to reallocate the lov target array. */ - struct lov_tgt_desc **newtgts, **old = NULL; - __u32 newsize, oldsize = 0; - - newsize = max(lov->lov_tgt_size, 2U); - while (newsize < index + 1) - newsize = newsize << 1; - OBD_ALLOC_PTR_ARRAY(newtgts, newsize); - if (newtgts == NULL) { - mutex_unlock(&lov->lov_lock); - RETURN(-ENOMEM); - } - - if (lov->lov_tgt_size) { - memcpy(newtgts, lov->lov_tgts, sizeof(*newtgts) * - lov->lov_tgt_size); - old = lov->lov_tgts; - oldsize = lov->lov_tgt_size; - } - - lov->lov_tgts = newtgts; - lov->lov_tgt_size = newsize; - smp_rmb(); - if (old) - OBD_FREE_PTR_ARRAY(old, oldsize); - - CDEBUG(D_CONFIG, "tgts: %p size: %d\n", - lov->lov_tgts, lov->lov_tgt_size); - } - OBD_ALLOC_PTR(tgt); if (!tgt) { - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); RETURN(-ENOMEM); } - rc = lu_tgt_pool_add(&lov->lov_packed, index, lov->lov_tgt_size); - if (rc) { - mutex_unlock(&lov->lov_lock); - OBD_FREE_PTR(tgt); - RETURN(rc); - } - + CDEBUG(D_CONFIG, "%s: Adding target %s\n", obd->obd_name, uuidp->uuid); tgt->ltd_uuid = *uuidp; tgt->ltd_obd = tgt_obd; /* XXX - add a sanity check on the generation number. */ tgt->ltd_gen = gen; tgt->ltd_index = index; tgt->ltd_activate = active; - lov->lov_tgts[index] = tgt; - if (index >= lov->desc.ld_tgt_count) - lov->desc.ld_tgt_count = index + 1; + rc = ltd_add_tgt(ltd, tgt); + if (rc < 0) { + mutex_unlock(<d->ltd_mutex); + OBD_FREE_PTR(tgt); + RETURN(rc); + } - mutex_unlock(&lov->lov_lock); + rc = lu_tgt_pool_add(<d->ltd_tgt_pool, index, ltd->ltd_tgts_size); + mutex_unlock(<d->ltd_mutex); + if (rc < 0) { + OBD_FREE_PTR(tgt); + RETURN(rc); + } CDEBUG(D_CONFIG, "idx=%d ltd_gen=%d ld_tgt_count=%d\n", - index, tgt->ltd_gen, lov->desc.ld_tgt_count); + index, tgt->ltd_gen, ltd->ltd_lov_desc.ld_tgt_count); if (lov->lov_connects == 0) { /* lov_connect hasn't been called yet. We'll do the * lov_connect_osc on this target when that fn first runs, - * because we don't know the connect flags yet. */ + * because we don't know the connect flags yet. + */ RETURN(0); } @@ -582,28 +555,29 @@ out: int lov_del_target(struct obd_device *obd, u32 index, struct obd_uuid *uuidp, int gen) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_obd *lov = &obd->u.lov; - int count = lov->desc.ld_tgt_count; + struct lov_tgt_desc *tgt; int rc = 0; - ENTRY; - if (index >= count) { + ENTRY; + if (index >= ltd->ltd_tgts_size || !test_bit(index, ltd->ltd_tgt_bitmap)) { CERROR("%s: LOV target index %d >= number of LOV OBDs %d: rc = %d\n", - obd->obd_name, index, count, -EINVAL); + obd->obd_name, index, ltd->ltd_tgts_size, -EINVAL); RETURN(-EINVAL); } /* to make sure there's no ongoing lov_notify() now */ down_write(&lov->lov_notify_lock); lov_tgts_getref(obd); - - if (!lov->lov_tgts[index]) { + tgt = lov_tgt(lov, index); + if (!tgt) { CERROR("%s: LOV target at index %d is not setup: rc = %d\n", obd->obd_name, index, -EINVAL); GOTO(out, rc = -EINVAL); } - if (uuidp && !obd_uuid_equals(uuidp, &lov->lov_tgts[index]->ltd_uuid)) { + if (uuidp && !obd_uuid_equals(uuidp, &tgt->ltd_uuid)) { CERROR("%s: LOV target UUID %s at index %d doesn't match %s: rc = %d\n", obd->obd_name, lov_uuid2str(lov, index), index, obd_uuid2str(uuidp), -EINVAL); @@ -612,12 +586,11 @@ int lov_del_target(struct obd_device *obd, u32 index, struct obd_uuid *uuidp, } CDEBUG(D_CONFIG, "uuid: %s idx: %d gen: %d exp: %p active: %d\n", - lov_uuid2str(lov, index), index, - lov->lov_tgts[index]->ltd_gen, lov->lov_tgts[index]->ltd_exp, - lov->lov_tgts[index]->ltd_active); + lov_uuid2str(lov, index), index, tgt->ltd_gen, tgt->ltd_exp, + tgt->ltd_active); - lov->lov_tgts[index]->ltd_reap = 1; - lov->lov_death_row++; + tgt->ltd_reap = 1; + ltd->ltd_death_row++; /* we really delete it from lov_tgts_putref() */ out: lov_tgts_putref(obd); @@ -645,7 +618,8 @@ static void __lov_del_obd(struct obd_device *obd, struct lov_tgt_desc *tgt) /* Manual cleanup - no cleanup logs to clean up the osc's. We must * do it ourselves. And we can't do it from lov_cleanup, - * because we just lost our only reference to it. */ + * because we just lost our only reference to it. + */ if (osc_obd) class_manual_cleanup(osc_obd); } @@ -697,6 +671,7 @@ void lov_fix_desc(struct lov_desc *desc) int lov_setup(struct obd_device *obd, struct lustre_cfg *lcfg) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_desc *desc; struct lov_obd *lov = &obd->u.lov; int rc; @@ -729,31 +704,40 @@ int lov_setup(struct obd_device *obd, struct lustre_cfg *lcfg) } } - lov_fix_desc(desc); + rc = lu_tgt_descs_init(ltd, false); + if (rc < 0) + RETURN(rc); + lov_fix_desc(desc); + desc->ld_tgt_count = 0; desc->ld_active_tgt_count = 0; - lov->desc = *desc; - lov->lov_tgt_size = 0; + ltd->ltd_lov_desc = *desc; - mutex_init(&lov->lov_lock); - atomic_set(&lov->lov_refcount, 0); lov->lov_sp_me = LUSTRE_SP_CLI; - init_rwsem(&lov->lov_notify_lock); INIT_LIST_HEAD(&lov->lov_pool_list); lov->lov_pool_count = 0; rc = lov_pool_hash_init(&lov->lov_pools_hash_body); - if (rc) + if (rc < 0) { + lu_tgt_descs_fini(ltd); GOTO(out, rc); + } - rc = lu_tgt_pool_init(&lov->lov_packed, 0); - if (rc) + rc = lu_tgt_pool_init(<d->ltd_tgt_pool, 0); + if (rc < 0) { + lov_pool_hash_destroy(&lov->lov_pools_hash_body); + lu_tgt_descs_fini(ltd); GOTO(out, rc); + } rc = lov_tunables_init(obd); - if (rc) + if (rc < 0) { + lu_tgt_pool_free(<d->ltd_tgt_pool); + lov_pool_hash_destroy(&lov->lov_pools_hash_body); + lu_tgt_descs_fini(ltd); GOTO(out, rc); + } lov->lov_tgts_kobj = kobject_create_and_add("target_obds", &obd->obd_kset.kobj); @@ -763,10 +747,12 @@ out: static int lov_cleanup(struct obd_device *obd) { + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; struct lov_obd *lov = &obd->u.lov; struct lov_pool_desc *pool, *tmp; - ENTRY; + struct lu_tgt_desc *tgt; + ENTRY; if (lov->lov_tgts_kobj) { kobject_put(lov->lov_tgts_kobj); lov->lov_tgts_kobj = NULL; @@ -780,32 +766,27 @@ static int lov_cleanup(struct obd_device *obd) lov_pool_del(obd, pool->pool_name); } lov_pool_hash_destroy(&lov->lov_pools_hash_body); - lu_tgt_pool_free(&lov->lov_packed); + lu_tgt_pool_free(<d->ltd_tgt_pool); lprocfs_obd_cleanup(obd); - if (lov->lov_tgts) { - int i; - lov_tgts_getref(obd); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - if (!lov->lov_tgts[i]) - continue; - - /* Inactive targets may never have connected */ - if (lov->lov_tgts[i]->ltd_active) - /* We should never get here - these - * should have been removed in the - * disconnect. */ - CERROR("%s: lov tgt %d not cleaned! " - "deathrow=%d, lovrc=%d\n", - obd->obd_name, i, lov->lov_death_row, - atomic_read(&lov->lov_refcount)); - lov_del_target(obd, i, NULL, 0); - } - lov_tgts_putref(obd); - OBD_FREE_PTR_ARRAY(lov->lov_tgts, lov->lov_tgt_size); - lov->lov_tgt_size = 0; + lov_tgts_getref(obd); + lov_foreach_tgt(lov, tgt) { + /* Inactive targets may never have connected */ + if (tgt->ltd_active) + /* We should never get here - these + * should have been removed in the + * disconnect. + */ + CERROR("%s: lov tgt %d not cleaned! deathrow=%d, lovrc=%d\n", + obd->obd_name, tgt->ltd_index, + ltd->ltd_death_row, + atomic_read(<d->ltd_refcount)); + lov_del_target(obd, tgt->ltd_index, NULL, 0); } + lov_tgts_putref(obd); + + lu_tgt_descs_fini(ltd); if (lov->lov_cache != NULL) { cl_cache_decref(lov->lov_cache); @@ -857,12 +838,8 @@ int lov_process_config_base(struct obd_device *obd, struct lustre_cfg *lcfg, GOTO(out, rc); } case LCFG_PARAM: { - struct lov_desc *desc = &(obd->u.lov.desc); ssize_t count; - if (!desc) - GOTO(out, rc = -EINVAL); - count = class_modify_config(lcfg, PARAM_LOV, &obd->obd_kset.kobj); GOTO(out, rc = count < 0 ? count : 0); @@ -899,7 +876,6 @@ static int lov_statfs(const struct lu_env *env, struct obd_export *exp, int rc2; ENTRY; - rqset = ptlrpc_prep_set(); if (rqset == NULL) RETURN(-ENOMEM); @@ -909,8 +885,10 @@ static int lov_statfs(const struct lu_env *env, struct obd_export *exp, GOTO(out_rqset, rc); list_for_each_entry(req, &set->set_list, rq_link) { - rc = obd_statfs_async(lov->lov_tgts[req->rq_idx]->ltd_exp, - &req->rq_oi, max_age, rqset); + struct lu_tgt_desc *tgt = lov_tgt(lov, req->rq_idx); + + rc = obd_statfs_async(tgt->ltd_exp, &req->rq_oi, max_age, + rqset); if (rc < 0) GOTO(out_set, rc); } @@ -936,7 +914,9 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, { struct obd_device *obd = class_exp2obd(exp); struct lov_obd *lov = &obd->u.lov; - int i = 0, rc = 0, count = lov->desc.ld_tgt_count; + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; + int rc = 0, count = ltd->ltd_tgts_size; + struct lov_tgt_desc *tgt = NULL; ENTRY; CDEBUG(D_IOCTL, "%s: cmd=%x len=%u karg=%pK uarg=%pK\n", @@ -968,17 +948,17 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, if (index >= count) RETURN(-ENODEV); - if (!lov->lov_tgts[index]) + if (!test_bit(index, lov->lov_ost_descs.ltd_tgt_bitmap)) /* Try again with the next index */ RETURN(-EAGAIN); - osc_obd = class_exp2obd(lov->lov_tgts[index]->ltd_exp); + tgt = lov_tgt(lov, index); + osc_obd = class_exp2obd(tgt->ltd_exp); if (!osc_obd) RETURN(-EINVAL); imp = osc_obd->u.cli.cl_import; - if (!lov->lov_tgts[index]->ltd_active && - imp->imp_state != LUSTRE_IMP_IDLE) + if (!tgt->ltd_active && imp->imp_state != LUSTRE_IMP_IDLE) RETURN(-ENODATA); /* copy UUID */ @@ -991,7 +971,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, flags = flags & LL_STATFS_NODELAY ? OBD_STATFS_NODELAY : 0; /* got statfs data */ - rc = obd_statfs(NULL, lov->lov_tgts[index]->ltd_exp, &stat_buf, + rc = obd_statfs(NULL, tgt->ltd_exp, &stat_buf, ktime_get_seconds() - OBD_STATFS_CACHE_SECONDS, flags); if (rc) @@ -1004,7 +984,6 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, } case OBD_IOC_QUOTACTL: { struct if_quotactl *qctl; - struct lov_tgt_desc *tgt = NULL; struct obd_quotactl *oqctl; struct obd_import *imp; @@ -1017,21 +996,20 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, if (count <= qctl->qc_idx) RETURN(-EINVAL); - tgt = lov->lov_tgts[qctl->qc_idx]; - if (!tgt) + if (!test_bit(qctl->qc_idx, + lov->lov_ost_descs.ltd_tgt_bitmap)) RETURN(-ENODEV); + tgt = lov_tgt(lov, qctl->qc_idx); if (!tgt->ltd_exp) RETURN(-EINVAL); } else if (qctl->qc_valid == QC_UUID) { - for (i = 0; i < count; i++) { - tgt = lov->lov_tgts[i]; - if (!tgt || - !obd_uuid_equals(&tgt->ltd_uuid, + lov_foreach_tgt(lov, tgt) { + if (!obd_uuid_equals(&tgt->ltd_uuid, &qctl->obd_uuid)) continue; - if (tgt->ltd_exp == NULL) + if (!tgt->ltd_exp) RETURN(-EINVAL); break; @@ -1040,7 +1018,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, RETURN(-EINVAL); } - if (i >= count) + if (!tgt || tgt->ltd_index >= count) RETURN(-EAGAIN); LASSERT(tgt && tgt->ltd_exp); @@ -1071,26 +1049,26 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, if (count == 0) RETURN(-ENOTTY); - for (i = 0; i < count; i++) { - int err; + lov_foreach_tgt(lov, tgt) { struct obd_device *osc_obd; + int err; /* OST was disconnected */ - if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_exp) + if (!tgt->ltd_exp) continue; /* ll_umount_begin() sets force on lov, pass to osc */ - osc_obd = class_exp2obd(lov->lov_tgts[i]->ltd_exp); + osc_obd = class_exp2obd(tgt->ltd_exp); if (osc_obd) osc_obd->obd_force = obd->obd_force; - err = obd_iocontrol(cmd, lov->lov_tgts[i]->ltd_exp, + err = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); if (err) { - if (lov->lov_tgts[i]->ltd_active) { + if (tgt->ltd_active) { OBD_IOC_DEBUG(err == -ENOTTY ? D_IOCTL : D_WARNING, obd->obd_name, cmd, - lov_uuid2str(lov, i), + lov_uuid2str(lov, tgt->ltd_index), err); if (!rc) rc = err; @@ -1115,11 +1093,11 @@ static int lov_get_info(const struct lu_env *env, struct obd_export *exp, __u32 keylen, void *key, __u32 *vallen, void *val) { struct obd_device *obd = class_exp2obd(exp); - struct lov_obd *lov = &obd->u.lov; - struct lov_desc *ld = &lov->desc; + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; + struct lov_desc *ld = <d->ltd_lov_desc; int rc = 0; - ENTRY; + ENTRY; if (vallen == NULL || val == NULL) RETURN(-EFAULT); @@ -1133,19 +1111,16 @@ static int lov_get_info(const struct lu_env *env, struct obd_export *exp, *((u32 *)val) = lov_mds_md_size(def_stripe_count, LOV_MAGIC_V3); } else if (KEY_IS(KEY_TGT_COUNT)) { - *((int *)val) = lov->desc.ld_tgt_count; + *((int *)val) = ld->ld_tgt_count; } else if (KEY_IS(KEY_MAX_PAGES_PER_RPC)) { struct lov_tgt_desc *tgt; struct client_obd *cli; struct obd_import *imp; u32 result = 0; - u32 i; - - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - tgt = lov->lov_tgts[i]; + lov_foreach_tgt(&obd->u.lov, tgt) { /* OST was disconnected */ - if (tgt == NULL || tgt->ltd_exp == NULL) + if (!tgt->ltd_exp) continue; cli = &tgt->ltd_obd->u.cli; @@ -1185,7 +1160,6 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp, int err; ENTRY; - if (set == NULL) { no_set = true; set = ptlrpc_prep_set(); @@ -1198,11 +1172,9 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp, if (KEY_IS(KEY_CHECKSUM)) do_inactive = true; - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - tgt = lov->lov_tgts[i]; - + lov_foreach_tgt(lov, tgt) { /* OST was disconnected */ - if (tgt == NULL || tgt->ltd_exp == NULL) + if (!tgt->ltd_exp) continue; /* OST is inactive and we don't want inactive OSCs */ @@ -1211,7 +1183,6 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp, err = obd_set_info_async(env, tgt->ltd_exp, keylen, key, vallen, val, set); - if (rc == 0) rc = err; } @@ -1266,7 +1237,7 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp, struct list_head *lst = NULL; __u64 curspace = 0; __u64 bhardlimit = 0; - int i, rc = 0; + int rc = 0; ENTRY; if (oqctl->qc_cmd != Q_GETOQUOTA && @@ -1293,26 +1264,22 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp, /* for lov tgt */ lov_tgts_getref(obd); - for (i = 0; i < lov->desc.ld_tgt_count; i++) { + lov_foreach_tgt(lov, tgt) { int err; - tgt = lov->lov_tgts[i]; - - if (!tgt) - continue; - if (pool && lu_tgt_check_index(tgt->ltd_index, &pool->pool_obds)) continue; if (!tgt->ltd_active || tgt->ltd_reap) { if (oqctl->qc_cmd == Q_GETOQUOTA && - lov->lov_tgts[i]->ltd_activate) { + tgt->ltd_activate) { rc = -ENETDOWN; CERROR("%s: ost %d is inactive: rc = %d\n", - obd->obd_name, i, rc); + obd->obd_name, tgt->ltd_index, rc); } else { - CDEBUG(D_HA, "ost %d is inactive\n", i); + CDEBUG(D_HA, "ost %d is inactive\n", + tgt->ltd_index); } continue; } diff --git a/lustre/lov/lov_object.c b/lustre/lov/lov_object.c index b9a3bbb..b39d0a4 100644 --- a/lustre/lov/lov_object.c +++ b/lustre/lov/lov_object.c @@ -218,7 +218,7 @@ static int lov_init_raid0(const struct lu_env *env, struct lov_device *dev, GOTO(out, result = -EIO); } - exp = dev->ld_lov->lov_tgts[ost_idx]->ltd_exp; + exp = lov_tgt(dev->ld_lov, ost_idx)->ltd_exp; if (likely(exp)) { /* the more fast OSTs the better */ if (exp->exp_obd->obd_osfs.os_state & OS_STATFS_NONROT) @@ -1840,20 +1840,24 @@ static int fiemap_for_stripe(const struct lu_env *env, struct cl_object *obj, switch (lle->lle_type) { case LOV_PATTERN_RAID0: { - struct lov_device *lov = lov_object_dev(lo); + struct lov_obd *lov = lu2lov_dev(obj->co_lu.lo_dev)->ld_lov; + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; const struct lov_layout_raid0 *r0 = &lle->lle_raid0; struct lov_oinfo *oinfo; + struct lu_tgt_desc *tgt; if (stripeno >= r0->lo_nr) RETURN(-EINVAL); + subobj = lovsub2cl(r0->lo_sub[stripeno]); oinfo = lsme->lsme_oinfo[stripeno]; if (lov_oinfo_is_dummy(oinfo)) RETURN(-EIO); devnr = oinfo->loi_ost_idx; - if (devnr < 0 || devnr >= lov_targets_nr(lov)) + if (devnr < 0 || devnr >= ltd->ltd_tgts_size) RETURN(-EINVAL); - if (!lov->ld_lov->lov_tgts[devnr]->ltd_active) { + tgt = lov_tgt(lov, devnr); + if (!tgt || !tgt->ltd_active) { ext_count = fiemap_unknown(fs, obd_start, obd_end); GOTO(out_unknown, rc = -ENODEV); } diff --git a/lustre/lov/lov_pack.c b/lustre/lov/lov_pack.c index 81c977b..9a0ebe1 100644 --- a/lustre/lov/lov_pack.c +++ b/lustre/lov/lov_pack.c @@ -286,12 +286,13 @@ ssize_t lov_lsm_pack(const struct lov_stripe_md *lsm, void *buf, /* Find the max stripecount we should use */ __u16 lov_get_stripe_count(struct lov_obd *lov, __u32 magic, __u16 stripe_count) { - __u32 max_stripes = LOV_MAX_STRIPE_COUNT_OLD; + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; + u32 max_stripes = LOV_MAX_STRIPE_COUNT_OLD; if (!stripe_count) - stripe_count = lov->desc.ld_default_stripe_count; - if (stripe_count > lov->desc.ld_active_tgt_count) - stripe_count = lov->desc.ld_active_tgt_count; + stripe_count = ltd->ltd_lov_desc.ld_default_stripe_count; + if (stripe_count > ltd->ltd_lov_desc.ld_active_tgt_count) + stripe_count = ltd->ltd_lov_desc.ld_active_tgt_count; if (!stripe_count) stripe_count = 1; diff --git a/lustre/lov/lov_pool.c b/lustre/lov/lov_pool.c index 500cfe3..e4e5c6c 100644 --- a/lustre/lov/lov_pool.c +++ b/lustre/lov/lov_pool.c @@ -27,7 +27,7 @@ #include "lov_internal.h" #define pool_tgt(_p, _i) \ - _p->pool_lobd->u.lov.lov_tgts[_p->pool_obds.op_array[_i]] + lov_tgt(&_p->pool_lobd->u.lov, _p->pool_obds.op_array[_i]) /** * Hash the pool name for use by the hashtable handlers. @@ -507,15 +507,13 @@ int lov_pool_del(struct obd_device *obd, char *poolname) */ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname) { + struct lov_obd *lov = &(obd->u.lov); struct obd_uuid ost_uuid; - struct lov_obd *lov; struct lov_pool_desc *pool; - unsigned int lov_idx; - int rc; - ENTRY; - - lov = &(obd->u.lov); + struct lu_tgt_desc *tgt; + int rc = -EINVAL; + ENTRY; rcu_read_lock(); pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname, pools_hash_params); @@ -529,18 +527,17 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname) /* search ost in lov array */ lov_tgts_getref(obd); - for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) { - if (!lov->lov_tgts[lov_idx]) - continue; - if (obd_uuid_equals(&ost_uuid, - &(lov->lov_tgts[lov_idx]->ltd_uuid))) + lov_foreach_tgt(lov, tgt) { + if (obd_uuid_equals(&ost_uuid, &tgt->ltd_uuid)) { + rc = 0; break; + } } - /* test if ost found in lov */ - if (lov_idx == lov->desc.ld_tgt_count) - GOTO(out, rc = -EINVAL); + if (rc < 0) + GOTO(out, rc); - rc = lu_tgt_pool_add(&pool->pool_obds, lov_idx, lov->lov_tgt_size); + rc = lu_tgt_pool_add(&pool->pool_obds, tgt->ltd_index, + lov->lov_ost_descs.ltd_tgts_size); if (rc) GOTO(out, rc); @@ -571,15 +568,13 @@ out: */ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname) { + struct lov_obd *lov = &(obd->u.lov); struct obd_uuid ost_uuid; - struct lov_obd *lov; struct lov_pool_desc *pool; - unsigned int lov_idx; - int rc = 0; - ENTRY; - - lov = &(obd->u.lov); + struct lu_tgt_desc *tgt; + int rc = -EINVAL; + ENTRY; /* lookup and kill hash reference */ rcu_read_lock(); pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname, @@ -594,20 +589,16 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname) lov_tgts_getref(obd); /* search ost in lov array, to get index */ - for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) { - if (!lov->lov_tgts[lov_idx]) - continue; - - if (obd_uuid_equals(&ost_uuid, - &(lov->lov_tgts[lov_idx]->ltd_uuid))) + lov_foreach_tgt(lov, tgt) { + if (obd_uuid_equals(&ost_uuid, &tgt->ltd_uuid)) { + rc = 0; break; + } } + if (rc < 0) + GOTO(out, rc); - /* test if ost found in lov */ - if (lov_idx == lov->desc.ld_tgt_count) - GOTO(out, rc = -EINVAL); - - lu_tgt_pool_remove(&pool->pool_obds, lov_idx); + lu_tgt_pool_remove(&pool->pool_obds, tgt->ltd_index); CDEBUG(D_CONFIG, "%s removed from "LOV_POOLNAMEF"\n", ostname, poolname); diff --git a/lustre/lov/lov_request.c b/lustre/lov/lov_request.c index e8e88bf..771f5b3 100644 --- a/lustre/lov/lov_request.c +++ b/lustre/lov/lov_request.c @@ -63,16 +63,19 @@ lov_set_add_req(struct lov_request *req, struct lov_request_set *set) static int lov_check_set(struct lov_obd *lov, int idx) { + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; + struct lu_tgt_desc *tgt; int rc = 0; - mutex_lock(&lov->lov_lock); + mutex_lock(<d->ltd_mutex); - if (!lov->lov_tgts[idx] || lov->lov_tgts[idx]->ltd_active || - (lov->lov_tgts[idx]->ltd_exp && - class_exp2cliimp(lov->lov_tgts[idx]->ltd_exp)->imp_connect_tried)) + tgt = lov_tgt(lov, idx); + if (!tgt || tgt->ltd_active || + (tgt->ltd_exp && + class_exp2cliimp(tgt->ltd_exp)->imp_connect_tried)) rc = 1; - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); return rc; } @@ -83,15 +86,15 @@ static int lov_check_set(struct lov_obd *lov, int idx) */ static int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx) { + struct lu_tgt_descs *ltd = &lov->lov_ost_descs; struct lov_tgt_desc *tgt; struct obd_import *imp = NULL; int rc = 0; int cnt; - mutex_lock(&lov->lov_lock); - - tgt = lov->lov_tgts[ost_idx]; + mutex_lock(<d->ltd_mutex); + tgt = lov_tgt(lov, ost_idx); if (unlikely(!tgt)) GOTO(out, rc = 0); @@ -105,7 +108,7 @@ static int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx) if (imp && imp->imp_state == LUSTRE_IMP_IDLE) GOTO(out, rc = 0); - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); cnt = obd_timeout; while (cnt > 0 && @@ -119,7 +122,7 @@ static int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx) return 0; out: - mutex_unlock(&lov->lov_lock); + mutex_unlock(<d->ltd_mutex); return rc; } @@ -270,7 +273,7 @@ static int cb_statfs_update(void *cookie, int rc) GOTO(out, rc); lov_tgts_getref(lovobd); - tgt = lov->lov_tgts[lovreq->rq_idx]; + tgt = lov_tgt(lov, lovreq->rq_idx); if (!tgt || !tgt->ltd_active) GOTO(out_update, rc); @@ -293,10 +296,10 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, { struct lov_request_set *set; struct lov_obd *lov = &obd->u.lov; - int rc = 0, i; + struct lu_tgt_desc *tgt; + int rc = 0; ENTRY; - OBD_ALLOC(set, sizeof(*set)); if (!set) RETURN(-ENOMEM); @@ -306,34 +309,28 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, set->set_oi = oinfo; /* We only get block data from the OBD */ - for (i = 0; i < lov->desc.ld_tgt_count; i++) { - struct lov_tgt_desc *ltd = lov->lov_tgts[i]; + lov_foreach_tgt(lov, tgt) { struct lov_request *req; - if (!ltd) { - CDEBUG(D_HA, "lov idx %d inactive\n", i); - continue; - } - /* * skip targets that have been explicitely disabled by the * administrator */ - if (!ltd->ltd_exp) { + if (!tgt->ltd_exp) { CDEBUG(D_HA, "lov idx %d administratively disabled\n", - i); + tgt->ltd_index); continue; } if (oinfo->oi_flags & OBD_STATFS_NODELAY && - class_exp2cliimp(ltd->ltd_exp)->imp_state != - LUSTRE_IMP_IDLE && !ltd->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", i); + class_exp2cliimp(tgt->ltd_exp)->imp_state != + LUSTRE_IMP_IDLE && !tgt->ltd_active) { + CDEBUG(D_HA, "lov idx %d inactive\n", tgt->ltd_index); continue; } - if (!ltd->ltd_active) - lov_check_and_wait_active(lov, i); + if (!tgt->ltd_active) + lov_check_and_wait_active(lov, tgt->ltd_index); OBD_ALLOC(req, sizeof(*req)); if (!req) @@ -345,7 +342,7 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, GOTO(out_set, rc = -ENOMEM); } - req->rq_idx = i; + req->rq_idx = tgt->ltd_index; req->rq_oi.oi_cb_up = cb_statfs_update; req->rq_oi.oi_flags = oinfo->oi_flags; diff --git a/lustre/lov/lproc_lov.c b/lustre/lov/lproc_lov.c index 7a67e20..699f8b8 100644 --- a/lustre/lov/lproc_lov.c +++ b/lustre/lov/lproc_lov.c @@ -25,7 +25,7 @@ static ssize_t stripesize_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return scnprintf(buf, PAGE_SIZE, "%llu\n", desc->ld_default_stripe_size); } @@ -35,7 +35,7 @@ static ssize_t stripesize_store(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; u64 val; int rc; @@ -55,7 +55,7 @@ static ssize_t stripeoffset_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%lld\n", desc->ld_default_stripe_offset); } @@ -65,7 +65,7 @@ static ssize_t stripeoffset_store(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; long val; int rc; @@ -86,7 +86,7 @@ static ssize_t stripetype_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%u\n", desc->ld_pattern); } @@ -96,7 +96,7 @@ static ssize_t stripetype_store(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; u32 pattern; int rc; @@ -116,7 +116,7 @@ static ssize_t stripecount_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%d\n", (__s16)(desc->ld_default_stripe_count + 1) - 1); @@ -127,7 +127,7 @@ static ssize_t stripecount_store(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; int stripe_count; int rc; @@ -150,7 +150,7 @@ static ssize_t numobd_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%u\n", desc->ld_tgt_count); } @@ -161,7 +161,7 @@ static ssize_t activeobd_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%u\n", desc->ld_active_tgt_count); } @@ -172,7 +172,7 @@ static ssize_t desc_uuid_show(struct kobject *kobj, struct attribute *attr, { struct obd_device *obd = container_of(kobj, struct obd_device, obd_kset.kobj); - struct lov_desc *desc = &obd->u.lov.desc; + struct lov_desc *desc = &obd->u.lov.lov_ost_descs.ltd_lov_desc; return sprintf(buf, "%s\n", desc->ld_uuid.uuid); } @@ -182,30 +182,42 @@ LUSTRE_RO_ATTR(desc_uuid); static void *lov_tgt_seq_start(struct seq_file *p, loff_t *pos) { struct obd_device *obd = p->private; - struct lov_obd *lov = &obd->u.lov; - - while (*pos < lov->desc.ld_tgt_count) { - if (lov->lov_tgts[*pos]) - return lov->lov_tgts[*pos]; - ++*pos; - } - return NULL; + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; + + lov_tgts_getref(obd); + if (*pos >= ltd->ltd_tgts_size) + return NULL; + + *pos = find_next_bit(ltd->ltd_tgt_bitmap, + ltd->ltd_tgts_size, *pos); + if (*pos < ltd->ltd_tgts_size) + return LTD_TGT(ltd, *pos); + else + return NULL; } static void lov_tgt_seq_stop(struct seq_file *p, void *v) { + struct obd_device *obd = p->private; + + lov_tgts_putref(obd); } static void *lov_tgt_seq_next(struct seq_file *p, void *v, loff_t *pos) { struct obd_device *obd = p->private; - struct lov_obd *lov = &obd->u.lov; - - while (++*pos < lov->desc.ld_tgt_count) { - if (lov->lov_tgts[*pos]) - return lov->lov_tgts[*pos]; - } - return NULL; + struct lu_tgt_descs *ltd = &obd->u.lov.lov_ost_descs; + + (*pos)++; + if (*pos > ltd->ltd_tgts_size - 1) + return NULL; + + *pos = find_next_bit(ltd->ltd_tgt_bitmap, + ltd->ltd_tgts_size, *pos); + if (*pos < ltd->ltd_tgts_size) + return LTD_TGT(ltd, *pos); + else + return NULL; } static int lov_tgt_seq_show(struct seq_file *p, void *v) diff --git a/lustre/obdclass/lu_tgt_descs.c b/lustre/obdclass/lu_tgt_descs.c index 03d9f54..ece3b57 100644 --- a/lustre/obdclass/lu_tgt_descs.c +++ b/lustre/obdclass/lu_tgt_descs.c @@ -242,7 +242,7 @@ int lu_tgt_descs_init(struct lu_tgt_descs *ltd, bool is_mdt) ltd->ltd_tgts_size = BITS_PER_LONG; ltd->ltd_death_row = 0; - ltd->ltd_refcount = 0; + atomic_set(<d->ltd_refcount, 0); /* Set up allocation policy (QoS and RR) */ INIT_LIST_HEAD(<d->ltd_qos.lq_svr_list); @@ -283,6 +283,7 @@ void lu_tgt_descs_fini(struct lu_tgt_descs *ltd) bitmap_free(ltd->ltd_tgt_bitmap); for (i = 0; i < ARRAY_SIZE(ltd->ltd_tgt_idx); i++) { OBD_FREE_PTR(ltd->ltd_tgt_idx[i]); + ltd->ltd_tgt_idx[i] = NULL; } ltd->ltd_tgts_size = 0; } diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index e7ccc91..a90eaa4 100755 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -7345,6 +7345,16 @@ restore_ostindex() { fi } +max_lov_stripe_index() { + locale facet=$1 + + if [[ $(lustre_version_code $facet) -lt $(version_code 2.15.64) ]]; then + echo "65532" + else + echo "65503" + fi +} + # The main purpose of this test is to ensure the OST_INDEX_LIST functions as # expected. This test uses OST_INDEX_LIST to format OSTs with a randomly # assigned index and ensures we can mount such a formatted file system @@ -7359,7 +7369,10 @@ test_81() { # LU-4665 # is generated. local i local saved_ostindex1=$OSTINDEX1 - for i in 65535 $((RANDOM + 65536)); do + local LOV_V1_INSANE_STRIPE_INDEX=$(max_lov_stripe_index ost1) + local invalid_index=$((LOV_V1_INSANE_STRIPE_INDEX+4)) + + for i in $((LOV_V1_INSANE_STRIPE_INDEX + 3)) $((RANDOM + invalid_index)); do echo -e "\nFormat ost1 with --index=$i, should fail" OSTINDEX1=$i if add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --reformat \ @@ -7374,9 +7387,11 @@ test_81() { # LU-4665 stack_trap restore_ostindex # Format OSTs with random sparse indices. - local rand_ost=$(((RANDOM * 2 % 65533) + 1)) - echo "Format $OSTCOUNT OSTs with OST_INDEX_LIST=[0,$rand_ost,65534]" - OST_INDEX_LIST=[0,$rand_ost,65534] formatall || + LOV_V1_INSANE_STRIPE_INDEX=$(max_lov_stripe_index ost2) + local rand_ost=$((RANDOM * 2 % (LOV_V1_INSANE_STRIPE_INDEX - 1) + 1)) + LOV_V1_INSANE_STRIPE_INDEX=$(max_lov_stripe_index ost3) + echo "Format $OSTCOUNT OSTs with OST_INDEX_LIST=[0,$rand_ost,$LOV_V1_INSANE_STRIPE_INDEX]" + OST_INDEX_LIST=[0,$rand_ost,$LOV_V1_INSANE_STRIPE_INDEX] formatall || error "formatall failed with $?" # Setup and check Lustre filesystem. @@ -7436,7 +7451,7 @@ run_test 81 "sparse OST indexing" random_ost_indices() { local num=$1 - local LOV_V1_INSANE_STRIPE_COUNT=65532 + local LOV_V1_INSANE_STRIPE_INDEX local index local skip local i=0 @@ -7444,7 +7459,8 @@ random_ost_indices() { while ((i < num)); do skip=false - index=$(((RANDOM * 2) % LOV_V1_INSANE_STRIPE_COUNT)) + LOV_V1_INSANE_STRIPE_INDEX=$(max_lov_stripe_index ost$((i+1))) + index=$(((RANDOM * 2) % LOV_V1_INSANE_STRIPE_INDEX)) for k in $ost_indices; do ((index == k)) && skip=true done @@ -9753,6 +9769,7 @@ test_110() fi fi + # Use INDEX_UNASSIGNED for index for --mkfsoptions if [[ $opts != *mkfsoptions* ]]; then opts+=" --mkfsoptions=\\\"-b 1024 -i 65536\\\"" else diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 7bae5a8..527f084 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -2507,7 +2507,7 @@ static int setup_indexes(int d, char *path, struct obd_uuid *obduuids, int num_obds, int **obdindexes, int *obdindex, enum tgt_type type) { - int ret, obdcount, maxidx, obd_valid = 0, obdnum; + int ret, obdcount, obd_valid = 0, obdnum; int *indices = NULL; struct obd_uuid *uuids = NULL; int *indexes; @@ -2527,7 +2527,6 @@ static int setup_indexes(int d, char *path, struct obd_uuid *obduuids, ret = -ENOMEM; goto out_uuids; } - maxidx = obdcount; retry_get_uuids: ret = llapi_get_target_uuids(d, uuids, indices, &obdcount, type); @@ -2560,14 +2559,16 @@ retry_get_uuids: } for (obdnum = 0; obdnum < num_obds; obdnum++) { + int maxidx = LOV_V1_INSANE_STRIPE_COUNT; char *end = NULL; /* The user may have specified a simple index */ i = strtol(obduuids[obdnum].uuid, &end, 0); - if (end && *end == '\0' && i < maxidx) { + if (end && *end == '\0' && i < LOV_V1_INSANE_STRIPE_COUNT) { indexes[obdnum] = i; obd_valid++; } else { + maxidx = obdcount; for (i = 0; i < obdcount; i++) { if (llapi_uuid_match(uuids[i].uuid, obduuids[obdnum].uuid)) { diff --git a/lustre/utils/mkfs_lustre.c b/lustre/utils/mkfs_lustre.c index 51d09cc..e1f54c3 100644 --- a/lustre/utils/mkfs_lustre.c +++ b/lustre/utils/mkfs_lustre.c @@ -449,11 +449,11 @@ static int parse_opts(int argc, char *const argv[], struct mkfs_opts *mop, progname, optarg); return 1; } - if (ldd->ldd_svindex >= INDEX_UNASSIGNED) { + if (ldd->ldd_svindex > LOV_V1_INSANE_STRIPE_INDEX) { fprintf(stderr, "%s: wrong index %u. Target index must be less than %u.\n", progname, ldd->ldd_svindex, - INDEX_UNASSIGNED); + LOV_V1_INSANE_STRIPE_INDEX + 1); return 1; }