From: Lai Siyao Date: Tue, 23 Apr 2019 18:46:05 +0000 (+0800) Subject: LU-11213 lmv: use lu_tgt_descs to manage tgts X-Git-Tag: 2.12.90~127 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=59fc1218fccf1a826182ff7cd52321e3efbb1eab LU-11213 lmv: use lu_tgt_descs to manage tgts Like LOD, use lu_tgt_descs to manage tgts, so that they can share tgt management code. TODO: use the same tgt management code for LOV/LFSCK. Signed-off-by: Lai Siyao Change-Id: I843e85e0f8b05e21056567cebfea0a8fff1d4ef8 Reviewed-on: https://review.whamcloud.com/35218 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Hongchao Zhang Reviewed-by: Oleg Drokin --- diff --git a/libcfs/include/libcfs/bitmap.h b/libcfs/include/libcfs/bitmap.h index 1763da2..b4782c4 100644 --- a/libcfs/include/libcfs/bitmap.h +++ b/libcfs/include/libcfs/bitmap.h @@ -41,8 +41,7 @@ struct cfs_bitmap { }; #define CFS_BITMAP_SIZE(nbits) \ - (((nbits / BITS_PER_LONG) + 1) * sizeof(long) + \ - sizeof(struct cfs_bitmap)) + (BITS_TO_LONGS(nbits) * sizeof(long) + sizeof(struct cfs_bitmap)) static inline struct cfs_bitmap *CFS_ALLOCATE_BITMAP(int size) diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 6f4c701..a81f9f9 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -1464,6 +1464,38 @@ struct lu_tgt_desc { ltd_connecting:1; /* target is connecting */ }; +/* number of pointers at 1st level */ +#define TGT_PTRS (PAGE_SIZE / sizeof(void *)) +/* number of pointers at 2nd level */ +#define TGT_PTRS_PER_BLOCK (PAGE_SIZE / sizeof(void *)) + +struct lu_tgt_desc_idx { + struct lu_tgt_desc *ldi_tgt[TGT_PTRS_PER_BLOCK]; +}; + +struct lu_tgt_descs { + /* list of known TGTs */ + struct lu_tgt_desc_idx *ltd_tgt_idx[TGT_PTRS]; + /* Size of the lu_tgts array, granted to be a power of 2 */ + __u32 ltd_tgts_size; + /* number of registered TGTs */ + __u32 ltd_tgtnr; + /* bitmap of TGTs available */ + struct cfs_bitmap *ltd_tgt_bitmap; + /* TGTs scheduled to be deleted */ + __u32 ltd_death_row; + /* Table refcount used for delayed deletion */ + int ltd_refcount; + /* mutex to serialize concurrent updates to the tgt table */ + struct mutex ltd_mutex; + /* read/write semaphore used for array relocation */ + struct rw_semaphore ltd_rw_sem; +}; + +#define LTD_TGT(ltd, index) \ + (ltd)->ltd_tgt_idx[(index) / \ + TGT_PTRS_PER_BLOCK]->ldi_tgt[(index) % TGT_PTRS_PER_BLOCK] + /* QoS data for LOD/LMV */ struct lu_qos { struct list_head lq_svr_list; /* lu_svr_qos list */ @@ -1483,5 +1515,41 @@ int lqos_add_tgt(struct lu_qos *qos, struct lu_tgt_desc *ltd); int lqos_del_tgt(struct lu_qos *qos, struct lu_tgt_desc *ltd); u64 lu_prandom_u64_max(u64 ep_ro); +int lu_tgt_descs_init(struct lu_tgt_descs *ltd); +void lu_tgt_descs_fini(struct lu_tgt_descs *ltd); +int lu_tgt_descs_add(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt); +void lu_tgt_descs_del(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt); + +static inline struct lu_tgt_desc *ltd_first_tgt(struct lu_tgt_descs *ltd) +{ + int index; + + index = find_first_bit(ltd->ltd_tgt_bitmap->data, + ltd->ltd_tgt_bitmap->size); + return (index < ltd->ltd_tgt_bitmap->size) ? LTD_TGT(ltd, index) : NULL; +} + +static inline struct lu_tgt_desc *ltd_next_tgt(struct lu_tgt_descs *ltd, + struct lu_tgt_desc *tgt) +{ + int index; + + if (!tgt) + return NULL; + + index = tgt->ltd_index; + LASSERT(index < ltd->ltd_tgt_bitmap->size); + index = find_next_bit(ltd->ltd_tgt_bitmap->data, + ltd->ltd_tgt_bitmap->size, index + 1); + return (index < ltd->ltd_tgt_bitmap->size) ? LTD_TGT(ltd, index) : NULL; +} + +#define ltd_foreach_tgt(ltd, tgt) \ + for (tgt = ltd_first_tgt(ltd); tgt; tgt = ltd_next_tgt(ltd, tgt)) + +#define ltd_foreach_tgt_safe(ltd, tgt, tmp) \ + for (tgt = ltd_first_tgt(ltd), tmp = ltd_next_tgt(ltd, tgt); tgt; \ + tgt = tmp, tmp = ltd_next_tgt(ltd, tgt)) + /** @} lu */ #endif /* __LUSTRE_LU_OBJECT_H */ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 702b0cc..f4cd024 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -427,14 +427,12 @@ struct lmv_obd { spinlock_t lmv_lock; struct lmv_desc desc; - struct mutex lmv_init_mutex; int connected; int max_easize; int max_def_easize; u32 lmv_statfs_start; - u32 tgts_size; /* size of tgts array */ - struct lmv_tgt_desc **tgts; + struct lu_tgt_descs lmv_mdt_descs; struct obd_connect_data conn_data; struct kobject *lmv_tgts_kobj; diff --git a/lustre/lfsck/lfsck_internal.h b/lustre/lfsck/lfsck_internal.h index 428e28f..e5b20e9 100644 --- a/lustre/lfsck/lfsck_internal.h +++ b/lustre/lfsck/lfsck_internal.h @@ -456,9 +456,6 @@ struct lfsck_operations { struct lfsck_start_param *lsp); }; -#define TGT_PTRS 256 /* number of pointers at 1st level */ -#define TGT_PTRS_PER_BLOCK 256 /* number of pointers at 2nd level */ - struct lfsck_tgt_desc { struct list_head ltd_orphan_list; struct dt_device *ltd_tgt; diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c index 92a2ff4..43c6b66 100644 --- a/lustre/lmv/lmv_intent.c +++ b/lustre/lmv/lmv_intent.c @@ -86,7 +86,7 @@ static int lmv_intent_remote(struct obd_export *exp, struct lookup_intent *it, LASSERT(fid_is_sane(&body->mbo_fid1)); - tgt = lmv_find_target(lmv, &body->mbo_fid1); + tgt = lmv_fid2tgt(lmv, &body->mbo_fid1); if (IS_ERR(tgt)) GOTO(out, rc = PTR_ERR(tgt)); @@ -201,9 +201,9 @@ int lmv_revalidate_slaves(struct obd_export *exp, op_data->op_fid1 = fid; op_data->op_fid2 = fid; - tgt = lmv_get_target(lmv, lsm->lsm_md_oinfo[i].lmo_mds, NULL); - if (IS_ERR(tgt)) - GOTO(cleanup, rc = PTR_ERR(tgt)); + tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[i].lmo_mds); + if (!tgt) + GOTO(cleanup, rc = -ENODEV); CDEBUG(D_INODE, "Revalidate slave "DFID" -> mds #%u\n", PFID(&fid), tgt->ltd_index); @@ -346,7 +346,7 @@ retry: if (lmv_dir_striped(op_data->op_mea1)) op_data->op_fid1 = op_data->op_fid2; - tgt = lmv_find_target(lmv, &op_data->op_fid2); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid2); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h index 2589f9f..56c389d 100644 --- a/lustre/lmv/lmv_internal.h +++ b/lustre/lmv/lmv_internal.h @@ -70,56 +70,81 @@ static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv) return container_of0(lmv, struct obd_device, u.lmv); } -static inline struct lmv_tgt_desc * -lmv_get_target(struct lmv_obd *lmv, u32 mdt_idx, int *index) +static inline struct lu_tgt_desc * +lmv_tgt(struct lmv_obd *lmv, __u32 index) { - int i; + return index < lmv->lmv_mdt_descs.ltd_tgt_bitmap->size ? + LTD_TGT(&lmv->lmv_mdt_descs, index) : NULL; +} - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - if (lmv->tgts[i] == NULL) - continue; +static inline bool +lmv_mdt0_inited(struct lmv_obd *lmv) +{ + return lmv->lmv_mdt_descs.ltd_tgt_bitmap->size > 0 && + cfs_bitmap_check(lmv->lmv_mdt_descs.ltd_tgt_bitmap, 0); +} - if (lmv->tgts[i]->ltd_index == mdt_idx) { - if (index != NULL) - *index = i; - return lmv->tgts[i]; - } - } +#define lmv_foreach_tgt(lmv, tgt) ltd_foreach_tgt(&(lmv)->lmv_mdt_descs, tgt) + +#define lmv_foreach_tgt_safe(lmv, tgt, tmp) \ + ltd_foreach_tgt_safe(&(lmv)->lmv_mdt_descs, tgt, tmp) + +static inline +struct lu_tgt_desc *lmv_first_connected_tgt(struct lmv_obd *lmv) +{ + struct lu_tgt_desc *tgt; - return ERR_PTR(-ENODEV); + tgt = ltd_first_tgt(&lmv->lmv_mdt_descs); + while (tgt && !tgt->ltd_exp) + tgt = ltd_next_tgt(&lmv->lmv_mdt_descs, tgt); + + return tgt; +} + +static inline +struct lu_tgt_desc *lmv_next_connected_tgt(struct lmv_obd *lmv, + struct lu_tgt_desc *tgt) +{ + do { + tgt = ltd_next_tgt(&lmv->lmv_mdt_descs, tgt); + } while (tgt && !tgt->ltd_exp); + + return tgt; } +#define lmv_foreach_connected_tgt(lmv, tgt) \ + for (tgt = lmv_first_connected_tgt(lmv); tgt; \ + tgt = lmv_next_connected_tgt(lmv, tgt)) + static inline int -lmv_find_target_index(struct lmv_obd *lmv, const struct lu_fid *fid) +lmv_fid2tgt_index(struct lmv_obd *lmv, const struct lu_fid *fid) { - struct lmv_tgt_desc *ltd; - u32 mdt_idx = 0; - int index = 0; - - if (lmv->desc.ld_tgt_count > 1) { - int rc; - rc = lmv_fld_lookup(lmv, fid, &mdt_idx); - if (rc < 0) - return rc; - } + u32 mdt_idx; + int rc; - ltd = lmv_get_target(lmv, mdt_idx, &index); - if (IS_ERR(ltd)) - return PTR_ERR(ltd); + if (lmv->desc.ld_tgt_count < 2) + return 0; - return index; + rc = lmv_fld_lookup(lmv, fid, &mdt_idx); + if (rc < 0) + return rc; + + return mdt_idx; } static inline struct lmv_tgt_desc * -lmv_find_target(struct lmv_obd *lmv, const struct lu_fid *fid) +lmv_fid2tgt(struct lmv_obd *lmv, const struct lu_fid *fid) { + struct lu_tgt_desc *tgt; int index; - index = lmv_find_target_index(lmv, fid); + index = lmv_fid2tgt_index(lmv, fid); if (index < 0) return ERR_PTR(index); - return lmv->tgts[index]; + tgt = lmv_tgt(lmv, index); + + return tgt ? tgt : ERR_PTR(-ENODEV); } static inline int lmv_stripe_md_size(int stripe_count) diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 7b12d26..6747995 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -83,50 +83,47 @@ static int lmv_set_mdc_active(struct lmv_obd *lmv, const struct obd_uuid *uuid, int activate) { - struct lmv_tgt_desc *tgt = NULL; - struct obd_device *obd; - __u32 i; - int rc = 0; + struct lu_tgt_desc *tgt = NULL; + struct obd_device *obd; + int rc = 0; + ENTRY; CDEBUG(D_INFO, "Searching in lmv %p for uuid %s (activate=%d)\n", lmv, uuid->uuid, activate); spin_lock(&lmv->lmv_lock); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; - - CDEBUG(D_INFO, "Target idx %d is %s conn %#llx\n", i, - tgt->ltd_uuid.uuid, tgt->ltd_exp->exp_handle.h_cookie); + lmv_foreach_connected_tgt(lmv, tgt) { + CDEBUG(D_INFO, "Target idx %d is %s conn %#llx\n", + tgt->ltd_index, tgt->ltd_uuid.uuid, + tgt->ltd_exp->exp_handle.h_cookie); if (obd_uuid_equals(uuid, &tgt->ltd_uuid)) break; } - if (i == lmv->desc.ld_tgt_count) - GOTO(out_lmv_lock, rc = -EINVAL); + if (!tgt) + GOTO(out_lmv_lock, rc = -EINVAL); - obd = class_exp2obd(tgt->ltd_exp); - if (obd == NULL) - GOTO(out_lmv_lock, rc = -ENOTCONN); + obd = class_exp2obd(tgt->ltd_exp); + if (obd == NULL) + GOTO(out_lmv_lock, rc = -ENOTCONN); - CDEBUG(D_INFO, "Found OBD %s=%s device %d (%p) type %s at LMV idx %d\n", - obd->obd_name, obd->obd_uuid.uuid, obd->obd_minor, obd, - obd->obd_type->typ_name, i); - LASSERT(strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) == 0); + CDEBUG(D_INFO, "Found OBD %s=%s device %d (%p) type %s at LMV idx %d\n", + obd->obd_name, obd->obd_uuid.uuid, obd->obd_minor, obd, + obd->obd_type->typ_name, tgt->ltd_index); + LASSERT(strcmp(obd->obd_type->typ_name, LUSTRE_MDC_NAME) == 0); - if (tgt->ltd_active == activate) { - CDEBUG(D_INFO, "OBD %p already %sactive!\n", obd, - activate ? "" : "in"); - GOTO(out_lmv_lock, rc); - } + if (tgt->ltd_active == activate) { + CDEBUG(D_INFO, "OBD %p already %sactive!\n", obd, + activate ? "" : "in"); + GOTO(out_lmv_lock, rc); + } - CDEBUG(D_INFO, "Marking OBD %p %sactive\n", obd, - activate ? "" : "in"); - lmv_activate_target(lmv, tgt, activate); - EXIT; + CDEBUG(D_INFO, "Marking OBD %p %sactive\n", obd, + activate ? "" : "in"); + lmv_activate_target(lmv, tgt, activate); + EXIT; out_lmv_lock: spin_unlock(&lmv->lmv_lock); @@ -135,8 +132,8 @@ static int lmv_set_mdc_active(struct lmv_obd *lmv, struct obd_uuid *lmv_get_uuid(struct obd_export *exp) { - struct lmv_obd *lmv = &exp->exp_obd->u.lmv; - struct lmv_tgt_desc *tgt = lmv->tgts[0]; + struct lmv_obd *lmv = &exp->exp_obd->u.lmv; + struct lmv_tgt_desc *tgt = lmv_tgt(lmv, 0); return (tgt == NULL) ? NULL : obd_get_uuid(tgt->ltd_exp); } @@ -240,21 +237,22 @@ out_sysfs: static int lmv_init_ea_size(struct obd_export *exp, __u32 easize, __u32 def_easize) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - __u32 i; - int rc = 0; - int change = 0; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int change = 0; + int rc = 0; + ENTRY; - if (lmv->max_easize < easize) { - lmv->max_easize = easize; - change = 1; - } - if (lmv->max_def_easize < def_easize) { - lmv->max_def_easize = def_easize; - change = 1; - } + if (lmv->max_easize < easize) { + lmv->max_easize = easize; + change = 1; + } + if (lmv->max_def_easize < def_easize) { + lmv->max_def_easize = def_easize; + change = 1; + } if (change == 0) RETURN(0); @@ -262,20 +260,14 @@ static int lmv_init_ea_size(struct obd_export *exp, __u32 easize, if (lmv->connected == 0) RETURN(0); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - struct lmv_tgt_desc *tgt = lmv->tgts[i]; - - if (tgt == NULL || tgt->ltd_exp == NULL) { - CWARN("%s: NULL export for %d\n", obd->obd_name, i); - continue; - } + lmv_foreach_connected_tgt(lmv, tgt) { if (!tgt->ltd_active) continue; rc = md_init_ea_size(tgt->ltd_exp, easize, def_easize); if (rc) { CERROR("%s: obd_init_ea_size() failed on MDT target %d:" - " rc = %d\n", obd->obd_name, i, rc); + " rc = %d\n", obd->obd_name, tgt->ltd_index, rc); break; } } @@ -375,16 +367,12 @@ int lmv_connect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt) RETURN(0); } -static void lmv_del_target(struct lmv_obd *lmv, int index) +static void lmv_del_target(struct lmv_obd *lmv, struct lu_tgt_desc *tgt) { - if (lmv->tgts[index] == NULL) - return; - - lqos_del_tgt(&lmv->lmv_qos, lmv->tgts[index]); - - OBD_FREE_PTR(lmv->tgts[index]); - lmv->tgts[index] = NULL; - return; + LASSERT(tgt); + lqos_del_tgt(&lmv->lmv_qos, tgt); + lu_tgt_descs_del(&lmv->lmv_mdt_descs, tgt); + OBD_FREE_PTR(tgt); } static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp, @@ -393,6 +381,7 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp, struct obd_device *mdc_obd; struct lmv_obd *lmv = &obd->u.lmv; struct lmv_tgt_desc *tgt; + struct lu_tgt_descs *ltd = &lmv->lmv_mdt_descs; int orig_tgt_count = 0; int rc = 0; @@ -407,78 +396,36 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp, RETURN(-EINVAL); } - mutex_lock(&lmv->lmv_init_mutex); - if ((index < lmv->tgts_size) && (lmv->tgts[index] != NULL)) { - tgt = lmv->tgts[index]; - CERROR("%s: UUID %s already assigned at LMV target index %d:" - " rc = %d\n", obd->obd_name, - obd_uuid2str(&tgt->ltd_uuid), index, -EEXIST); - mutex_unlock(&lmv->lmv_init_mutex); - RETURN(-EEXIST); - } - - if (index >= lmv->tgts_size) { - /* We need to reallocate the lmv target array. */ - struct lmv_tgt_desc **newtgts, **old = NULL; - __u32 newsize = 1; - __u32 oldsize = 0; - - while (newsize < index + 1) - newsize = newsize << 1; - OBD_ALLOC(newtgts, sizeof(*newtgts) * newsize); - if (newtgts == NULL) { - mutex_unlock(&lmv->lmv_init_mutex); - RETURN(-ENOMEM); - } - - if (lmv->tgts_size) { - memcpy(newtgts, lmv->tgts, - sizeof(*newtgts) * lmv->tgts_size); - old = lmv->tgts; - oldsize = lmv->tgts_size; - } - - lmv->tgts = newtgts; - lmv->tgts_size = newsize; - smp_rmb(); - if (old) - OBD_FREE(old, sizeof(*old) * oldsize); - - CDEBUG(D_CONFIG, "tgts: %p size: %d\n", lmv->tgts, - lmv->tgts_size); - } - OBD_ALLOC_PTR(tgt); - if (!tgt) { - mutex_unlock(&lmv->lmv_init_mutex); + if (!tgt) RETURN(-ENOMEM); - } mutex_init(&tgt->ltd_fid_mutex); tgt->ltd_index = index; tgt->ltd_uuid = *uuidp; tgt->ltd_active = 0; - lmv->tgts[index] = tgt; - if (index >= lmv->desc.ld_tgt_count) { + + mutex_lock(<d->ltd_mutex); + rc = lu_tgt_descs_add(ltd, tgt); + if (!rc && index >= lmv->desc.ld_tgt_count) { orig_tgt_count = lmv->desc.ld_tgt_count; lmv->desc.ld_tgt_count = index + 1; } + mutex_unlock(<d->ltd_mutex); + + if (rc) + GOTO(out_tgt, rc); - if (lmv->connected == 0) { + if (!lmv->connected) /* lmv_check_connect() will connect this target. */ - mutex_unlock(&lmv->lmv_init_mutex); RETURN(0); - } - /* Otherwise let's connect it ourselves */ - mutex_unlock(&lmv->lmv_init_mutex); rc = lmv_connect_mdc(obd, tgt); if (rc != 0) { - spin_lock(&lmv->lmv_lock); - if (lmv->desc.ld_tgt_count == index + 1) - lmv->desc.ld_tgt_count = orig_tgt_count; + mutex_lock(<d->ltd_mutex); + lmv->desc.ld_tgt_count = orig_tgt_count; memset(tgt, 0, sizeof(*tgt)); - spin_unlock(&lmv->lmv_lock); + mutex_unlock(<d->ltd_mutex); } else { int easize = sizeof(struct lmv_stripe_md) + lmv->desc.ld_tgt_count * sizeof(struct lu_fid); @@ -486,48 +433,44 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp, } RETURN(rc); + +out_tgt: + OBD_FREE_PTR(tgt); + return rc; } static int lmv_check_connect(struct obd_device *obd) { - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - __u32 i; - int rc; - int easize; - ENTRY; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int easize; + int rc; - if (lmv->connected) - RETURN(0); + ENTRY; - mutex_lock(&lmv->lmv_init_mutex); - if (lmv->connected) { - mutex_unlock(&lmv->lmv_init_mutex); - RETURN(0); - } + if (lmv->connected) + RETURN(0); - if (lmv->desc.ld_tgt_count == 0) { - mutex_unlock(&lmv->lmv_init_mutex); - CERROR("%s: no targets configured.\n", obd->obd_name); - RETURN(-EINVAL); - } + mutex_lock(&lmv->lmv_mdt_descs.ltd_mutex); + if (lmv->connected) + GOTO(unlock, rc = 0); - LASSERT(lmv->tgts != NULL); + if (lmv->desc.ld_tgt_count == 0) { + CERROR("%s: no targets configured: rc = -EINVAL\n", + obd->obd_name); + GOTO(unlock, rc = -EINVAL); + } - if (lmv->tgts[0] == NULL) { - mutex_unlock(&lmv->lmv_init_mutex); - CERROR("%s: no target configured for index 0.\n", + if (!lmv_mdt0_inited(lmv)) { + CERROR("%s: no target configured for index 0: rc = -EINVAL.\n", obd->obd_name); - RETURN(-EINVAL); + GOTO(unlock, rc = -EINVAL); } CDEBUG(D_CONFIG, "Time to connect %s to %s\n", obd->obd_uuid.uuid, obd->obd_name); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - if (tgt == NULL) - continue; + lmv_foreach_tgt(lmv, tgt) { rc = lmv_connect_mdc(obd, tgt); if (rc) GOTO(out_disc, rc); @@ -536,30 +479,23 @@ static int lmv_check_connect(struct obd_device *obd) lmv->connected = 1; easize = lmv_mds_md_size(lmv->desc.ld_tgt_count, LMV_MAGIC); lmv_init_ea_size(obd->obd_self_export, easize, 0); - mutex_unlock(&lmv->lmv_init_mutex); - RETURN(0); + EXIT; +unlock: + mutex_unlock(&lmv->lmv_mdt_descs.ltd_mutex); - out_disc: - while (i-- > 0) { - int rc2; - tgt = lmv->tgts[i]; - if (tgt == NULL) + return rc; + +out_disc: + lmv_foreach_tgt(lmv, tgt) { + tgt->ltd_active = 0; + if (!tgt->ltd_exp) continue; - tgt->ltd_active = 0; - if (tgt->ltd_exp) { - --lmv->desc.ld_active_tgt_count; - rc2 = obd_disconnect(tgt->ltd_exp); - if (rc2) { - CERROR("LMV target %s disconnect on " - "MDC idx %d: error %d\n", - tgt->ltd_uuid.uuid, i, rc2); - } - } - } - mutex_unlock(&lmv->lmv_init_mutex); + --lmv->desc.ld_active_tgt_count; + obd_disconnect(tgt->ltd_exp); + } - RETURN(rc); + goto unlock; } static int lmv_disconnect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt) @@ -608,33 +544,22 @@ static int lmv_disconnect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt) static int lmv_disconnect(struct obd_export *exp) { - struct obd_device *obd = class_exp2obd(exp); - struct lmv_obd *lmv = &obd->u.lmv; - int rc; - __u32 i; - ENTRY; - - if (!lmv->tgts) - goto out_local; + struct obd_device *obd = class_exp2obd(exp); + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - if (lmv->tgts[i] == NULL || lmv->tgts[i]->ltd_exp == NULL) - continue; + ENTRY; - lmv_disconnect_mdc(obd, lmv->tgts[i]); - } + lmv_foreach_connected_tgt(lmv, tgt) + lmv_disconnect_mdc(obd, tgt); if (lmv->lmv_tgts_kobj) kobject_put(lmv->lmv_tgts_kobj); -out_local: - /* - * This is the case when no real connection is established by - * lmv_check_connect(). - */ - if (!lmv->connected) - class_export_put(exp); - rc = class_disconnect(exp); + if (!lmv->connected) + class_export_put(exp); + rc = class_disconnect(exp); lmv->connected = 0; RETURN(rc); @@ -643,17 +568,17 @@ out_local: static int lmv_fid2path(struct obd_export *exp, int len, void *karg, void __user *uarg) { - struct obd_device *obddev = class_exp2obd(exp); - struct lmv_obd *lmv = &obddev->u.lmv; + struct obd_device *obddev = class_exp2obd(exp); + struct lmv_obd *lmv = &obddev->u.lmv; struct getinfo_fid2path *gf; - struct lmv_tgt_desc *tgt; + struct lmv_tgt_desc *tgt; struct getinfo_fid2path *remote_gf = NULL; - struct lu_fid root_fid; - int remote_gf_size = 0; - int rc; + struct lu_fid root_fid; + int remote_gf_size = 0; + int rc; gf = karg; - tgt = lmv_find_target(lmv, &gf->gf_fid); + tgt = lmv_fid2tgt(lmv, &gf->gf_fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -711,7 +636,7 @@ repeat_fid2path: GOTO(out_fid2path, rc = -EINVAL); } - tgt = lmv_find_target(lmv, &gf->gf_fid); + tgt = lmv_fid2tgt(lmv, &gf->gf_fid); if (IS_ERR(tgt)) GOTO(out_fid2path, rc = -EINVAL); @@ -733,13 +658,13 @@ static int lmv_hsm_req_count(struct lmv_obd *lmv, const struct hsm_user_request *hur, const struct lmv_tgt_desc *tgt_mds) { - __u32 i; - int nr = 0; - struct lmv_tgt_desc *curr_tgt; + struct lmv_tgt_desc *curr_tgt; + __u32 i; + int nr = 0; /* count how many requests must be sent to the given target */ for (i = 0; i < hur->hur_request.hr_itemcount; i++) { - curr_tgt = lmv_find_target(lmv, &hur->hur_user_item[i].hui_fid); + curr_tgt = lmv_fid2tgt(lmv, &hur->hur_user_item[i].hui_fid); if (IS_ERR(curr_tgt)) RETURN(PTR_ERR(curr_tgt)); if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid)) @@ -753,15 +678,14 @@ static int lmv_hsm_req_build(struct lmv_obd *lmv, const struct lmv_tgt_desc *tgt_mds, struct hsm_user_request *hur_out) { - __u32 i, nr_out; - struct lmv_tgt_desc *curr_tgt; + __u32 i, nr_out; + struct lmv_tgt_desc *curr_tgt; /* build the hsm_user_request for the given target */ hur_out->hur_request = hur_in->hur_request; nr_out = 0; for (i = 0; i < hur_in->hur_request.hr_itemcount; i++) { - curr_tgt = lmv_find_target(lmv, - &hur_in->hur_user_item[i].hui_fid); + curr_tgt = lmv_fid2tgt(lmv, &hur_in->hur_user_item[i].hui_fid); if (IS_ERR(curr_tgt)) RETURN(PTR_ERR(curr_tgt)); if (obd_uuid_equals(&curr_tgt->ltd_uuid, &tgt_mds->ltd_uuid)) { @@ -782,20 +706,16 @@ static int lmv_hsm_ct_unregister(struct obd_device *obd, unsigned int cmd, void __user *uarg) { struct lmv_obd *lmv = &obd->u.lmv; - __u32 i; - int rc; + struct lu_tgt_desc *tgt; + int rc; + ENTRY; /* unregister request (call from llapi_hsm_copytool_fini) */ - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - struct lmv_tgt_desc *tgt = lmv->tgts[i]; - - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; + lmv_foreach_connected_tgt(lmv, tgt) /* best effort: try to clean as much as possible * (continue on error) */ obd_iocontrol(cmd, tgt->ltd_exp, len, lk, uarg); - } /* Whatever the result, remove copytool from kuc groups. * Unreached coordinators will get EPIPE on next requests @@ -812,12 +732,14 @@ static int lmv_hsm_ct_register(struct obd_device *obd, unsigned int cmd, { struct lmv_obd *lmv = &obd->u.lmv; struct file *filp; - __u32 i, j; - int err; bool any_set = false; struct kkuc_ct_data *kcd; size_t kcd_size; + struct lu_tgt_desc *tgt; + __u32 i; + int err; int rc = 0; + ENTRY; filp = fget(lk->lk_wfd); @@ -853,12 +775,7 @@ static int lmv_hsm_ct_register(struct obd_device *obd, unsigned int cmd, /* All or nothing: try to register to all MDS. * In case of failure, unregister from previous MDS, * except if it because of inactive target. */ - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - struct lmv_tgt_desc *tgt = lmv->tgts[i]; - - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; - + lmv_foreach_connected_tgt(lmv, tgt) { err = obd_iocontrol(cmd, tgt->ltd_exp, len, lk, uarg); if (err) { if (tgt->ltd_active) { @@ -866,14 +783,16 @@ static int lmv_hsm_ct_register(struct obd_device *obd, unsigned int cmd, CERROR("%s: iocontrol MDC %s on MDT" " idx %d cmd %x: err = %d\n", lmv2obd_dev(lmv)->obd_name, - tgt->ltd_uuid.uuid, i, cmd, err); + tgt->ltd_uuid.uuid, tgt->ltd_index, cmd, + err); rc = err; lk->lk_flags |= LK_FLG_STOP; + i = tgt->ltd_index; /* unregister from previous MDS */ - for (j = 0; j < i; j++) { - tgt = lmv->tgts[j]; - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; + lmv_foreach_connected_tgt(lmv, tgt) { + if (tgt->ltd_index >= i) + break; + obd_iocontrol(cmd, tgt->ltd_exp, len, lk, uarg); } @@ -901,37 +820,34 @@ err_fput: return rc; } - - - static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, int len, void *karg, void __user *uarg) { - struct obd_device *obddev = class_exp2obd(exp); - struct lmv_obd *lmv = &obddev->u.lmv; - struct lmv_tgt_desc *tgt = NULL; - __u32 i = 0; - int rc = 0; - int set = 0; - __u32 count = lmv->desc.ld_tgt_count; + struct obd_device *obddev = class_exp2obd(exp); + struct lmv_obd *lmv = &obddev->u.lmv; + struct lu_tgt_desc *tgt = NULL; + int set = 0; + __u32 count = lmv->desc.ld_tgt_count; + int rc = 0; + ENTRY; - if (count == 0) - RETURN(-ENOTTY); + if (count == 0) + RETURN(-ENOTTY); - switch (cmd) { - case IOC_OBD_STATFS: { - struct obd_ioctl_data *data = karg; - struct obd_device *mdc_obd; - struct obd_statfs stat_buf = {0}; - __u32 index; + switch (cmd) { + case IOC_OBD_STATFS: { + struct obd_ioctl_data *data = karg; + struct obd_device *mdc_obd; + struct obd_statfs stat_buf = {0}; + __u32 index; - memcpy(&index, data->ioc_inlbuf2, sizeof(__u32)); - if ((index >= count)) - RETURN(-ENODEV); + memcpy(&index, data->ioc_inlbuf2, sizeof(__u32)); + if (index >= count) + RETURN(-ENODEV); - tgt = lmv->tgts[index]; - if (tgt == NULL || !tgt->ltd_active) + tgt = lmv_tgt(lmv, index); + if (!tgt || !tgt->ltd_active) RETURN(-ENODATA); mdc_obd = class_exp2obd(tgt->ltd_exp); @@ -954,59 +870,56 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, (int) sizeof(stat_buf)))) RETURN(-EFAULT); break; - } - case OBD_IOC_QUOTACTL: { - struct if_quotactl *qctl = karg; - struct obd_quotactl *oqctl; + } + case OBD_IOC_QUOTACTL: { + struct if_quotactl *qctl = karg; + struct obd_quotactl *oqctl; if (qctl->qc_valid == QC_MDTIDX) { if (count <= qctl->qc_idx) RETURN(-EINVAL); - tgt = lmv->tgts[qctl->qc_idx]; - if (tgt == NULL || tgt->ltd_exp == NULL) + tgt = lmv_tgt(lmv, qctl->qc_idx); + if (!tgt || !tgt->ltd_exp) RETURN(-EINVAL); } else if (qctl->qc_valid == QC_UUID) { - for (i = 0; i < count; i++) { - tgt = lmv->tgts[i]; - if (tgt == NULL) - continue; + lmv_foreach_tgt(lmv, tgt) { if (!obd_uuid_equals(&tgt->ltd_uuid, &qctl->obd_uuid)) continue; - if (tgt->ltd_exp == NULL) - RETURN(-EINVAL); + if (!tgt->ltd_exp) + RETURN(-EINVAL); - break; - } - } else { - RETURN(-EINVAL); - } + break; + } + } else { + RETURN(-EINVAL); + } - if (i >= count) - RETURN(-EAGAIN); + if (tgt->ltd_index >= count) + RETURN(-EAGAIN); - LASSERT(tgt != NULL && tgt->ltd_exp != NULL); - OBD_ALLOC_PTR(oqctl); - if (!oqctl) - RETURN(-ENOMEM); + LASSERT(tgt != NULL && tgt->ltd_exp != NULL); + OBD_ALLOC_PTR(oqctl); + if (!oqctl) + RETURN(-ENOMEM); - QCTL_COPY(oqctl, qctl); - rc = obd_quotactl(tgt->ltd_exp, oqctl); - if (rc == 0) { - QCTL_COPY(qctl, oqctl); - qctl->qc_valid = QC_MDTIDX; - qctl->obd_uuid = tgt->ltd_uuid; - } - OBD_FREE_PTR(oqctl); - break; - } + QCTL_COPY(oqctl, qctl); + rc = obd_quotactl(tgt->ltd_exp, oqctl); + if (rc == 0) { + QCTL_COPY(qctl, oqctl); + qctl->qc_valid = QC_MDTIDX; + qctl->obd_uuid = tgt->ltd_uuid; + } + OBD_FREE_PTR(oqctl); + break; + } case LL_IOC_GET_CONNECT_FLAGS: { - tgt = lmv->tgts[0]; - if (tgt == NULL || tgt->ltd_exp == NULL) - RETURN(-ENODATA); - rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); + tgt = lmv_tgt(lmv, 0); + rc = -ENODATA; + if (tgt && tgt->ltd_exp) + rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); break; } case LL_IOC_FID2MDTIDX: { @@ -1029,9 +942,9 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, case LL_IOC_HSM_STATE_GET: case LL_IOC_HSM_STATE_SET: case LL_IOC_HSM_ACTION: { - struct md_op_data *op_data = karg; + struct md_op_data *op_data = karg; - tgt = lmv_find_target(lmv, &op_data->op_fid1); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -1044,7 +957,7 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, case LL_IOC_HSM_PROGRESS: { const struct hsm_progress_kernel *hpk = karg; - tgt = lmv_find_target(lmv, &hpk->hpk_fid); + tgt = lmv_fid2tgt(lmv, &hpk->hpk_fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); @@ -1061,22 +974,17 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp, * or if there is a single MDS, no need to split * the request. */ if (reqcount == 1 || count == 1) { - tgt = lmv_find_target(lmv, - &hur->hur_user_item[0].hui_fid); + tgt = lmv_fid2tgt(lmv, &hur->hur_user_item[0].hui_fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg); } else { /* split fid list to their respective MDS */ - for (i = 0; i < count; i++) { + lmv_foreach_connected_tgt(lmv, tgt) { int nr, rc1; size_t reqlen; struct hsm_user_request *req; - tgt = lmv->tgts[i]; - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; - nr = lmv_hsm_req_count(lmv, hur, tgt); if (nr < 0) RETURN(nr); @@ -1104,14 +1012,14 @@ hsm_req_err: break; } case LL_IOC_LOV_SWAP_LAYOUTS: { - struct md_op_data *op_data = karg; - struct lmv_tgt_desc *tgt1, *tgt2; + struct md_op_data *op_data = karg; + struct lmv_tgt_desc *tgt1, *tgt2; - tgt1 = lmv_find_target(lmv, &op_data->op_fid1); + tgt1 = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(tgt1)) RETURN(PTR_ERR(tgt1)); - tgt2 = lmv_find_target(lmv, &op_data->op_fid2); + tgt2 = lmv_fid2tgt(lmv, &op_data->op_fid2); if (IS_ERR(tgt2)) RETURN(PTR_ERR(tgt2)); @@ -1134,13 +1042,10 @@ hsm_req_err: break; } default: - for (i = 0; i < count; i++) { + lmv_foreach_connected_tgt(lmv, tgt) { struct obd_device *mdc_obd; int err; - tgt = lmv->tgts[i]; - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; /* ll_umount_begin() sets force flag but for lmv, not * mdc. Let's pass it through */ mdc_obd = class_exp2obd(tgt->ltd_exp); @@ -1150,17 +1055,18 @@ hsm_req_err: if (tgt->ltd_active) { CERROR("error: iocontrol MDC %s on MDT" " idx %d cmd %x: err = %d\n", - tgt->ltd_uuid.uuid, i, cmd, err); + tgt->ltd_uuid.uuid, + tgt->ltd_index, cmd, err); if (!rc) rc = err; } } else set = 1; - } - if (!set && !rc) - rc = -EIO; - } - RETURN(rc); + } + if (!set && !rc) + rc = -EIO; + } + RETURN(rc); } /** @@ -1216,13 +1122,14 @@ static u32 lmv_placement_policy(struct obd_device *obd, int __lmv_fid_alloc(struct lmv_obd *lmv, struct lu_fid *fid, u32 mds) { - struct lmv_tgt_desc *tgt; - int rc; + struct lmv_tgt_desc *tgt; + int rc; + ENTRY; - tgt = lmv_get_target(lmv, mds, NULL); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_tgt(lmv, mds); + if (!tgt) + RETURN(-ENODEV); /* * New seq alloc and FLD setup should be atomic. Otherwise we may find @@ -1280,22 +1187,17 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg) ENTRY; - if (LUSTRE_CFG_BUFLEN(lcfg, 1) < 1) { - CERROR("LMV setup requires a descriptor\n"); - RETURN(-EINVAL); - } - - desc = (struct lmv_desc *)lustre_cfg_buf(lcfg, 1); - if (sizeof(*desc) > LUSTRE_CFG_BUFLEN(lcfg, 1)) { - CERROR("Lmv descriptor size wrong: %d > %d\n", - (int)sizeof(*desc), LUSTRE_CFG_BUFLEN(lcfg, 1)); - RETURN(-EINVAL); - } + if (LUSTRE_CFG_BUFLEN(lcfg, 1) < 1) { + CERROR("LMV setup requires a descriptor\n"); + RETURN(-EINVAL); + } - lmv->tgts_size = 32U; - OBD_ALLOC(lmv->tgts, sizeof(*lmv->tgts) * lmv->tgts_size); - if (lmv->tgts == NULL) - RETURN(-ENOMEM); + desc = (struct lmv_desc *)lustre_cfg_buf(lcfg, 1); + if (sizeof(*desc) > LUSTRE_CFG_BUFLEN(lcfg, 1)) { + CERROR("Lmv descriptor size wrong: %d > %d\n", + (int)sizeof(*desc), LUSTRE_CFG_BUFLEN(lcfg, 1)); + RETURN(-EINVAL); + } obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid); lmv->desc.ld_tgt_count = 0; @@ -1305,7 +1207,6 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg) lmv->max_easize = 0; spin_lock_init(&lmv->lmv_lock); - mutex_init(&lmv->lmv_init_mutex); /* Set up allocation policy (QoS and RR) */ INIT_LIST_HEAD(&lmv->lmv_qos.lq_svr_list); @@ -1337,33 +1238,30 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg) rc = fld_client_init(&lmv->lmv_fld, obd->obd_name, LUSTRE_CLI_FLD_HASH_DHT); - if (rc) { + if (rc) CERROR("Can't init FLD, err %d\n", rc); - GOTO(out, rc); - } - RETURN(0); + rc = lu_tgt_descs_init(&lmv->lmv_mdt_descs); + if (rc) + CWARN("%s: error initialize target table: rc = %d\n", + obd->obd_name, rc); -out: - return rc; + RETURN(rc); } static int lmv_cleanup(struct obd_device *obd) { - struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_obd *lmv = &obd->u.lmv; + struct lu_tgt_desc *tgt; + struct lu_tgt_desc *tmp; + ENTRY; fld_client_fini(&lmv->lmv_fld); - if (lmv->tgts != NULL) { - int i; - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - if (lmv->tgts[i] == NULL) - continue; - lmv_del_target(lmv, i); - } - OBD_FREE(lmv->tgts, sizeof(*lmv->tgts) * lmv->tgts_size); - lmv->tgts_size = 0; - } + lmv_foreach_tgt_safe(lmv, tgt, tmp) + lmv_del_target(lmv, tgt); + lu_tgt_descs_fini(&lmv->lmv_mdt_descs); + RETURN(0); } @@ -1431,31 +1329,33 @@ static int lmv_select_statfs_mdt(struct lmv_obd *lmv, __u32 flags) static int lmv_statfs(const struct lu_env *env, struct obd_export *exp, struct obd_statfs *osfs, time64_t max_age, __u32 flags) { - struct obd_device *obd = class_exp2obd(exp); - struct lmv_obd *lmv = &obd->u.lmv; - struct obd_statfs *temp; - int rc = 0; - __u32 i, idx; + struct obd_device *obd = class_exp2obd(exp); + struct lmv_obd *lmv = &obd->u.lmv; + struct obd_statfs *temp; + struct lu_tgt_desc *tgt; + __u32 i; + __u32 idx; + int rc = 0; + ENTRY; - OBD_ALLOC(temp, sizeof(*temp)); - if (temp == NULL) - RETURN(-ENOMEM); + OBD_ALLOC(temp, sizeof(*temp)); + if (temp == NULL) + RETURN(-ENOMEM); /* distribute statfs among MDTs */ idx = lmv_select_statfs_mdt(lmv, flags); for (i = 0; i < lmv->desc.ld_tgt_count; i++, idx++) { idx = idx % lmv->desc.ld_tgt_count; - if (lmv->tgts[idx] == NULL || lmv->tgts[idx]->ltd_exp == NULL) + tgt = lmv_tgt(lmv, idx); + if (!tgt || !tgt->ltd_exp) continue; - rc = obd_statfs(env, lmv->tgts[idx]->ltd_exp, temp, - max_age, flags); + rc = obd_statfs(env, tgt->ltd_exp, temp, max_age, flags); if (rc) { CERROR("%s: can't stat MDS #%d: rc = %d\n", - lmv->tgts[idx]->ltd_exp->exp_obd->obd_name, i, - rc); + tgt->ltd_exp->exp_obd->obd_name, i, rc); GOTO(out_free_temp, rc); } @@ -1481,12 +1381,12 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp, osfs->os_files += temp->os_files; osfs->os_granted += temp->os_granted; } - } + } - EXIT; + EXIT; out_free_temp: - OBD_FREE(temp, sizeof(*temp)); - return rc; + OBD_FREE(temp, sizeof(*temp)); + return rc; } static int lmv_statfs_update(void *cookie, int rc) @@ -1534,12 +1434,17 @@ int lmv_statfs_check_update(struct obd_device *obd, struct lmv_tgt_desc *tgt) static int lmv_get_root(struct obd_export *exp, const char *fileset, struct lu_fid *fid) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lu_tgt_desc *tgt = lmv_tgt(lmv, 0); + int rc; - rc = md_get_root(lmv->tgts[0]->ltd_exp, fileset, fid); + ENTRY; + + if (!tgt) + RETURN(-ENODEV); + + rc = md_get_root(tgt->ltd_exp, fileset, fid); RETURN(rc); } @@ -1547,15 +1452,16 @@ static int lmv_getxattr(struct obd_export *exp, const struct lu_fid *fid, u64 obd_md_valid, const char *name, size_t buf_size, struct ptlrpc_request **req) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + + ENTRY; - tgt = lmv_find_target(lmv, fid); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_fid2tgt(lmv, fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); rc = md_getxattr(tgt->ltd_exp, fid, obd_md_valid, name, buf_size, req); @@ -1568,15 +1474,16 @@ static int lmv_setxattr(struct obd_export *exp, const struct lu_fid *fid, unsigned int xattr_flags, u32 suppgid, struct ptlrpc_request **req) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + + ENTRY; - tgt = lmv_find_target(lmv, fid); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_fid2tgt(lmv, fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); rc = md_setxattr(tgt->ltd_exp, fid, obd_md_valid, name, value, value_size, xattr_flags, suppgid, req); @@ -1585,67 +1492,67 @@ static int lmv_setxattr(struct obd_export *exp, const struct lu_fid *fid, } static int lmv_getattr(struct obd_export *exp, struct md_op_data *op_data, - struct ptlrpc_request **request) + struct ptlrpc_request **request) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + + ENTRY; - tgt = lmv_find_target(lmv, &op_data->op_fid1); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); if (op_data->op_flags & MF_GET_MDT_IDX) { op_data->op_mds = tgt->ltd_index; RETURN(0); } - rc = md_getattr(tgt->ltd_exp, op_data, request); + rc = md_getattr(tgt->ltd_exp, op_data, request); - RETURN(rc); + RETURN(rc); } static int lmv_null_inode(struct obd_export *exp, const struct lu_fid *fid) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - __u32 i; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lu_tgt_desc *tgt; + + ENTRY; - CDEBUG(D_INODE, "CBDATA for "DFID"\n", PFID(fid)); + CDEBUG(D_INODE, "CBDATA for "DFID"\n", PFID(fid)); /* * With DNE every object can have two locks in different namespaces: * lookup lock in space of MDT storing direntry and update/open lock in * space of MDT storing inode. */ - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - if (lmv->tgts[i] == NULL || lmv->tgts[i]->ltd_exp == NULL) - continue; - md_null_inode(lmv->tgts[i]->ltd_exp, fid); - } + lmv_foreach_connected_tgt(lmv, tgt) + md_null_inode(tgt->ltd_exp, fid); RETURN(0); } static int lmv_close(struct obd_export *exp, struct md_op_data *op_data, - struct md_open_data *mod, struct ptlrpc_request **request) + struct md_open_data *mod, struct ptlrpc_request **request) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; - tgt = lmv_find_target(lmv, &op_data->op_fid1); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + ENTRY; - CDEBUG(D_INODE, "CLOSE "DFID"\n", PFID(&op_data->op_fid1)); - rc = md_close(tgt->ltd_exp, op_data, mod, request); - RETURN(rc); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + + CDEBUG(D_INODE, "CLOSE "DFID"\n", PFID(&op_data->op_fid1)); + rc = md_close(tgt->ltd_exp, op_data, mod, request); + RETURN(rc); } static struct lmv_tgt_desc * @@ -1657,7 +1564,7 @@ lmv_locate_tgt_by_name(struct lmv_obd *lmv, struct lmv_stripe_md *lsm, const struct lmv_oinfo *oinfo; if (!lmv_dir_striped(lsm) || !namelen) { - tgt = lmv_find_target(lmv, fid); + tgt = lmv_fid2tgt(lmv, fid); if (IS_ERR(tgt)) return tgt; @@ -1678,11 +1585,11 @@ lmv_locate_tgt_by_name(struct lmv_obd *lmv, struct lmv_stripe_md *lsm, *fid = oinfo->lmo_fid; *mds = oinfo->lmo_mds; - tgt = lmv_get_target(lmv, oinfo->lmo_mds, NULL); + tgt = lmv_tgt(lmv, oinfo->lmo_mds); CDEBUG(D_INODE, "locate MDT %u parent "DFID"\n", *mds, PFID(fid)); - return tgt; + return tgt ? tgt : ERR_PTR(-ENODEV); } /** @@ -1718,9 +1625,9 @@ lmv_locate_tgt(struct lmv_obd *lmv, struct md_op_data *op_data) * ct_restore(). */ if (op_data->op_bias & MDS_CREATE_VOLATILE && (int)op_data->op_mds != -1) { - tgt = lmv_get_target(lmv, op_data->op_mds, NULL); - if (IS_ERR(tgt)) - return tgt; + tgt = lmv_tgt(lmv, op_data->op_mds); + if (!tgt) + return ERR_PTR(-ENODEV); if (lmv_dir_striped(lsm)) { int i; @@ -1743,7 +1650,9 @@ lmv_locate_tgt(struct lmv_obd *lmv, struct md_op_data *op_data) op_data->op_fid1 = oinfo->lmo_fid; op_data->op_mds = oinfo->lmo_mds; - tgt = lmv_get_target(lmv, oinfo->lmo_mds, NULL); + tgt = lmv_tgt(lmv, oinfo->lmo_mds); + if (!tgt) + tgt = ERR_PTR(-ENODEV); } else if (op_data->op_code == LUSTRE_OPC_MKDIR && lmv_dir_qos_mkdir(op_data->op_default_mea1) && !lmv_dir_striped(lsm)) { @@ -1875,7 +1784,7 @@ int lmv_create(struct obd_export *exp, struct md_op_data *op_data, if (exp_connect_flags(exp) & OBD_CONNECT_DIR_STRIPE) { /* Send the create request to the MDT where the object * will be located */ - tgt = lmv_find_target(lmv, &op_data->op_fid2); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid2); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -1901,15 +1810,16 @@ lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo, const union ldlm_policy_data *policy, struct md_op_data *op_data, struct lustre_handle *lockh, __u64 extra_lock_flags) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + ENTRY; CDEBUG(D_INODE, "ENQUEUE on "DFID"\n", PFID(&op_data->op_fid1)); - tgt = lmv_find_target(lmv, &op_data->op_fid1); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -1992,7 +1902,7 @@ static int lmv_early_cancel(struct obd_export *exp, struct lmv_tgt_desc *tgt, RETURN(0); if (tgt == NULL) { - tgt = lmv_find_target(lmv, fid); + tgt = lmv_fid2tgt(lmv, fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); } @@ -2080,7 +1990,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data, op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid()); op_data->op_cap = cfs_curproc_cap_pack(); - parent_tgt = lmv_find_target(lmv, &op_data->op_fid1); + parent_tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(parent_tgt)) RETURN(PTR_ERR(parent_tgt)); @@ -2107,10 +2017,9 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data, /* save it in fid4 temporarily for early cancel */ op_data->op_fid4 = lsm->lsm_md_oinfo[rc].lmo_fid; - sp_tgt = lmv_get_target(lmv, lsm->lsm_md_oinfo[rc].lmo_mds, - NULL); - if (IS_ERR(sp_tgt)) - RETURN(PTR_ERR(sp_tgt)); + sp_tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[rc].lmo_mds); + if (!sp_tgt) + RETURN(-ENODEV); /* * if parent is being migrated too, fill op_fid2 with target @@ -2127,17 +2036,15 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data, RETURN(rc); op_data->op_fid2 = lsm->lsm_md_oinfo[rc].lmo_fid; - tp_tgt = lmv_get_target(lmv, - lsm->lsm_md_oinfo[rc].lmo_mds, - NULL); - if (IS_ERR(tp_tgt)) - RETURN(PTR_ERR(tp_tgt)); + tp_tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[rc].lmo_mds); + if (!tp_tgt) + RETURN(-ENODEV); } } else { sp_tgt = parent_tgt; } - child_tgt = lmv_find_target(lmv, &op_data->op_fid3); + child_tgt = lmv_fid2tgt(lmv, &op_data->op_fid3); if (IS_ERR(child_tgt)) RETURN(PTR_ERR(child_tgt)); @@ -2160,7 +2067,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data, */ if (S_ISDIR(op_data->op_mode) && (exp_connect_flags2(exp) & OBD_CONNECT2_DIR_MIGRATE)) { - tgt = lmv_find_target(lmv, &target_fid); + tgt = lmv_fid2tgt(lmv, &target_fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); } else { @@ -2258,7 +2165,7 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data, * target child does not exist, then it will send the request to the * target parent */ if (fid_is_sane(&op_data->op_fid4)) { - tgt = lmv_find_target(lmv, &op_data->op_fid4); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid4); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); } else { @@ -2286,7 +2193,7 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data, } if (fid_is_sane(&op_data->op_fid3)) { - src_tgt = lmv_find_target(lmv, &op_data->op_fid3); + src_tgt = lmv_fid2tgt(lmv, &op_data->op_fid3); if (IS_ERR(src_tgt)) RETURN(PTR_ERR(src_tgt)); @@ -2352,7 +2259,7 @@ rename: ptlrpc_req_finished(*request); *request = NULL; - tgt = lmv_find_target(lmv, &op_data->op_fid4); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid4); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2374,10 +2281,11 @@ rename: static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data, void *ea, size_t ealen, struct ptlrpc_request **request) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc = 0; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc = 0; + ENTRY; CDEBUG(D_INODE, "SETATTR for "DFID", valid 0x%x/0x%x\n", @@ -2385,7 +2293,7 @@ static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data, op_data->op_xvalid); op_data->op_flags |= MF_MDC_CANCEL_FID1; - tgt = lmv_find_target(lmv, &op_data->op_fid1); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2397,13 +2305,14 @@ static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data, static int lmv_fsync(struct obd_export *exp, const struct lu_fid *fid, struct ptlrpc_request **request) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + ENTRY; - tgt = lmv_find_target(lmv, fid); + tgt = lmv_fid2tgt(lmv, fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2512,9 +2421,9 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt, break; } - tgt = lmv_get_target(ctxt->ldc_lmv, oinfo->lmo_mds, NULL); - if (IS_ERR(tgt)) { - rc = PTR_ERR(tgt); + tgt = lmv_tgt(ctxt->ldc_lmv, oinfo->lmo_mds); + if (!tgt) { + rc = -ENODEV; break; } @@ -2555,17 +2464,18 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt, static int lmv_file_resync(struct obd_export *exp, struct md_op_data *data) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; + ENTRY; rc = lmv_check_connect(obd); if (rc != 0) RETURN(rc); - tgt = lmv_find_target(lmv, &data->op_fid1); + tgt = lmv_fid2tgt(lmv, &data->op_fid1); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2786,7 +2696,7 @@ int lmv_read_page(struct obd_export *exp, struct md_op_data *op_data, RETURN(rc); } - tgt = lmv_find_target(lmv, &op_data->op_fid1); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2841,7 +2751,7 @@ retry: RETURN(PTR_ERR(parent_tgt)); if (likely(!fid_is_zero(&op_data->op_fid2))) { - tgt = lmv_find_target(lmv, &op_data->op_fid2); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid2); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); } else { @@ -2894,7 +2804,7 @@ retry: ptlrpc_req_finished(*request); *request = NULL; - tgt = lmv_find_target(lmv, &op_data->op_fid2); + tgt = lmv_fid2tgt(lmv, &op_data->op_fid2); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -2930,31 +2840,24 @@ static int lmv_precleanup(struct obd_device *obd) static int lmv_get_info(const struct lu_env *env, struct obd_export *exp, __u32 keylen, void *key, __u32 *vallen, void *val) { - struct obd_device *obd; - struct lmv_obd *lmv; - int rc = 0; - ENTRY; - - obd = class_exp2obd(exp); - if (obd == NULL) { - CDEBUG(D_IOCTL, "Invalid client cookie %#llx\n", - exp->exp_handle.h_cookie); - RETURN(-EINVAL); - } + struct obd_device *obd; + struct lmv_obd *lmv; + struct lu_tgt_desc *tgt; + int rc = 0; - lmv = &obd->u.lmv; - if (keylen >= strlen("remote_flag") && !strcmp(key, "remote_flag")) { - int i; + ENTRY; - LASSERT(*vallen == sizeof(__u32)); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - struct lmv_tgt_desc *tgt = lmv->tgts[i]; - /* - * All tgts should be connected when this gets called. - */ - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; + obd = class_exp2obd(exp); + if (obd == NULL) { + CDEBUG(D_IOCTL, "Invalid client cookie %#llx\n", + exp->exp_handle.h_cookie); + RETURN(-EINVAL); + } + lmv = &obd->u.lmv; + if (keylen >= strlen("remote_flag") && !strcmp(key, "remote_flag")) { + LASSERT(*vallen == sizeof(__u32)); + lmv_foreach_connected_tgt(lmv, tgt) { if (!obd_get_info(env, tgt->ltd_exp, keylen, key, vallen, val)) RETURN(0); @@ -2967,18 +2870,21 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp, * Forwarding this request to first MDS, it should know LOV * desc. */ - rc = obd_get_info(env, lmv->tgts[0]->ltd_exp, keylen, key, - vallen, val); + tgt = lmv_tgt(lmv, 0); + if (!tgt) + RETURN(-ENODEV); + + rc = obd_get_info(env, tgt->ltd_exp, keylen, key, vallen, val); if (!rc && KEY_IS(KEY_CONN_DATA)) exp->exp_connect_data = *(struct obd_connect_data *)val; - RETURN(rc); - } else if (KEY_IS(KEY_TGT_COUNT)) { - *((int *)val) = lmv->desc.ld_tgt_count; - RETURN(0); - } + RETURN(rc); + } else if (KEY_IS(KEY_TGT_COUNT)) { + *((int *)val) = lmv->desc.ld_tgt_count; + RETURN(0); + } - CDEBUG(D_IOCTL, "Invalid key\n"); - RETURN(-EINVAL); + CDEBUG(D_IOCTL, "Invalid key\n"); + RETURN(-EINVAL); } static int lmv_rmfid(struct obd_export *exp, struct fid_array *fa, @@ -2988,6 +2894,7 @@ static int lmv_rmfid(struct obd_export *exp, struct fid_array *fa, struct ptlrpc_request_set *set = _set; struct lmv_obd *lmv = &obddev->u.lmv; int tgt_count = lmv->desc.ld_tgt_count; + struct lu_tgt_desc *tgt; struct fid_array *fat, **fas = NULL; int i, rc, **rcs = NULL; @@ -3029,11 +2936,11 @@ static int lmv_rmfid(struct obd_export *exp, struct fid_array *fa, fat->fa_fids[fat->fa_nr++] = fa->fa_fids[i]; } - for (i = 0; i < tgt_count; i++) { - fat = fas[i]; + lmv_foreach_connected_tgt(lmv, tgt) { + fat = fas[tgt->ltd_index]; if (!fat || fat->fa_nr == 0) continue; - rc = md_rmfid(lmv->tgts[i]->ltd_exp, fat, rcs[i], set); + rc = md_rmfid(tgt->ltd_exp, fat, rcs[tgt->ltd_index], set); } rc = ptlrpc_set_wait(NULL, set); @@ -3107,14 +3014,9 @@ int lmv_set_info_async(const struct lu_env *env, struct obd_export *exp, if (KEY_IS(KEY_READ_ONLY) || KEY_IS(KEY_FLUSH_CTX) || KEY_IS(KEY_DEFAULT_EASIZE)) { - int i, err = 0; - - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - - if (tgt == NULL || tgt->ltd_exp == NULL) - continue; + int err = 0; + lmv_foreach_connected_tgt(lmv, tgt) { err = obd_set_info_async(env, tgt->ltd_exp, keylen, key, vallen, val, set); if (err && rc == 0) @@ -3319,17 +3221,16 @@ static int lmv_cancel_unused(struct obd_export *exp, const struct lu_fid *fid, void *opaque) { struct lmv_obd *lmv = &exp->exp_obd->u.lmv; + struct lu_tgt_desc *tgt; + int err; int rc = 0; - __u32 i; + ENTRY; LASSERT(fid != NULL); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - struct lmv_tgt_desc *tgt = lmv->tgts[i]; - int err; - - if (tgt == NULL || tgt->ltd_exp == NULL || !tgt->ltd_active) + lmv_foreach_connected_tgt(lmv, tgt) { + if (!tgt->ltd_active) continue; err = md_cancel_unused(tgt->ltd_exp, fid, policy, mode, flags, @@ -3344,9 +3245,10 @@ static int lmv_set_lock_data(struct obd_export *exp, const struct lustre_handle *lockh, void *data, __u64 *bits) { - struct lmv_obd *lmv = &exp->exp_obd->u.lmv; - struct lmv_tgt_desc *tgt = lmv->tgts[0]; - int rc; + struct lmv_obd *lmv = &exp->exp_obd->u.lmv; + struct lmv_tgt_desc *tgt = lmv_tgt(lmv, 0); + int rc; + ENTRY; if (tgt == NULL || tgt->ltd_exp == NULL) @@ -3360,37 +3262,38 @@ enum ldlm_mode lmv_lock_match(struct obd_export *exp, __u64 flags, union ldlm_policy_data *policy, enum ldlm_mode mode, struct lustre_handle *lockh) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - enum ldlm_mode rc; - int tgt; - int i; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + enum ldlm_mode rc; + struct lu_tgt_desc *tgt; + int i; + int index; + ENTRY; CDEBUG(D_INODE, "Lock match for "DFID"\n", PFID(fid)); - /* + /* * With DNE every object can have two locks in different namespaces: * lookup lock in space of MDT storing direntry and update/open lock in * space of MDT storing inode. Try the MDT that the FID maps to first, * since this can be easily found, and only try others if that fails. */ - for (i = 0, tgt = lmv_find_target_index(lmv, fid); + for (i = 0, index = lmv_fid2tgt_index(lmv, fid); i < lmv->desc.ld_tgt_count; - i++, tgt = (tgt + 1) % lmv->desc.ld_tgt_count) { - if (tgt < 0) { + i++, index = (index + 1) % lmv->desc.ld_tgt_count) { + if (index < 0) { CDEBUG(D_HA, "%s: "DFID" is inaccessible: rc = %d\n", - obd->obd_name, PFID(fid), tgt); - tgt = 0; + obd->obd_name, PFID(fid), index); + index = 0; } - if (lmv->tgts[tgt] == NULL || - lmv->tgts[tgt]->ltd_exp == NULL || - lmv->tgts[tgt]->ltd_active == 0) + tgt = lmv_tgt(lmv, index); + if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) continue; - rc = md_lock_match(lmv->tgts[tgt]->ltd_exp, flags, fid, - type, policy, mode, lockh); + rc = md_lock_match(tgt->ltd_exp, flags, fid, type, policy, mode, + lockh); if (rc) RETURN(rc); } @@ -3402,20 +3305,21 @@ int lmv_get_lustre_md(struct obd_export *exp, struct ptlrpc_request *req, struct obd_export *dt_exp, struct obd_export *md_exp, struct lustre_md *md) { - struct lmv_obd *lmv = &exp->exp_obd->u.lmv; - struct lmv_tgt_desc *tgt = lmv->tgts[0]; + struct lmv_obd *lmv = &exp->exp_obd->u.lmv; + struct lmv_tgt_desc *tgt = lmv_tgt(lmv, 0); - if (tgt == NULL || tgt->ltd_exp == NULL) - RETURN(-EINVAL); + if (!tgt || !tgt->ltd_exp) + return -EINVAL; - return md_get_lustre_md(lmv->tgts[0]->ltd_exp, req, dt_exp, md_exp, md); + return md_get_lustre_md(tgt->ltd_exp, req, dt_exp, md_exp, md); } int lmv_free_lustre_md(struct obd_export *exp, struct lustre_md *md) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt = lmv->tgts[0]; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt = lmv_tgt(lmv, 0); + ENTRY; if (md->default_lmv) { @@ -3426,21 +3330,22 @@ int lmv_free_lustre_md(struct obd_export *exp, struct lustre_md *md) lmv_free_memmd(md->lmv); md->lmv = NULL; } - if (tgt == NULL || tgt->ltd_exp == NULL) + if (!tgt || !tgt->ltd_exp) RETURN(-EINVAL); - RETURN(md_free_lustre_md(lmv->tgts[0]->ltd_exp, md)); + RETURN(md_free_lustre_md(tgt->ltd_exp, md)); } int lmv_set_open_replay_data(struct obd_export *exp, struct obd_client_handle *och, struct lookup_intent *it) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + ENTRY; - tgt = lmv_find_target(lmv, &och->och_fid); + tgt = lmv_fid2tgt(lmv, &och->och_fid); if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); @@ -3448,18 +3353,19 @@ int lmv_set_open_replay_data(struct obd_export *exp, } int lmv_clear_open_replay_data(struct obd_export *exp, - struct obd_client_handle *och) + struct obd_client_handle *och) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + + ENTRY; - tgt = lmv_find_target(lmv, &och->och_fid); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + tgt = lmv_fid2tgt(lmv, &och->och_fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); - RETURN(md_clear_open_replay_data(tgt->ltd_exp, och)); + RETURN(md_clear_open_replay_data(tgt->ltd_exp, och)); } int lmv_intent_getattr_async(struct obd_export *exp, @@ -3481,7 +3387,7 @@ int lmv_intent_getattr_async(struct obd_export *exp, if (IS_ERR(ptgt)) RETURN(PTR_ERR(ptgt)); - ctgt = lmv_find_target(lmv, &op_data->op_fid2); + ctgt = lmv_fid2tgt(lmv, &op_data->op_fid1); if (IS_ERR(ctgt)) RETURN(PTR_ERR(ctgt)); @@ -3498,20 +3404,21 @@ int lmv_intent_getattr_async(struct obd_export *exp, } int lmv_revalidate_lock(struct obd_export *exp, struct lookup_intent *it, - struct lu_fid *fid, __u64 *bits) + struct lu_fid *fid, __u64 *bits) { - struct obd_device *obd = exp->exp_obd; - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt; - int rc; - ENTRY; + struct obd_device *obd = exp->exp_obd; + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt; + int rc; - tgt = lmv_find_target(lmv, fid); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + ENTRY; - rc = md_revalidate_lock(tgt->ltd_exp, it, fid, bits); - RETURN(rc); + tgt = lmv_fid2tgt(lmv, fid); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + + rc = md_revalidate_lock(tgt->ltd_exp, it, fid, bits); + RETURN(rc); } int lmv_get_fid_from_lsm(struct obd_export *exp, @@ -3539,49 +3446,46 @@ int lmv_get_fid_from_lsm(struct obd_export *exp, int lmv_quotactl(struct obd_device *unused, struct obd_export *exp, struct obd_quotactl *oqctl) { - struct obd_device *obd = class_exp2obd(exp); - struct lmv_obd *lmv = &obd->u.lmv; - struct lmv_tgt_desc *tgt = lmv->tgts[0]; - int rc = 0; - __u32 i; - __u64 curspace, curinodes; + struct obd_device *obd = class_exp2obd(exp); + struct lmv_obd *lmv = &obd->u.lmv; + struct lmv_tgt_desc *tgt = lmv_tgt(lmv, 0); + __u64 curspace, curinodes; + int rc = 0; + ENTRY; - if (tgt == NULL || - tgt->ltd_exp == NULL || - !tgt->ltd_active || - lmv->desc.ld_tgt_count == 0) { + if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) { CERROR("master lmv inactive\n"); RETURN(-EIO); } - if (oqctl->qc_cmd != Q_GETOQUOTA) { - rc = obd_quotactl(tgt->ltd_exp, oqctl); - RETURN(rc); - } + if (oqctl->qc_cmd != Q_GETOQUOTA) { + rc = obd_quotactl(tgt->ltd_exp, oqctl); + RETURN(rc); + } - curspace = curinodes = 0; - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { + curspace = curinodes = 0; + lmv_foreach_connected_tgt(lmv, tgt) { int err; - tgt = lmv->tgts[i]; - if (tgt == NULL || tgt->ltd_exp == NULL || !tgt->ltd_active) + if (!tgt->ltd_active) continue; - err = obd_quotactl(tgt->ltd_exp, oqctl); - if (err) { - CERROR("getquota on mdt %d failed. %d\n", i, err); - if (!rc) - rc = err; - } else { - curspace += oqctl->qc_dqblk.dqb_curspace; - curinodes += oqctl->qc_dqblk.dqb_curinodes; - } - } - oqctl->qc_dqblk.dqb_curspace = curspace; - oqctl->qc_dqblk.dqb_curinodes = curinodes; + err = obd_quotactl(tgt->ltd_exp, oqctl); + if (err) { + CERROR("getquota on mdt %d failed. %d\n", + tgt->ltd_index, err); + if (!rc) + rc = err; + } else { + curspace += oqctl->qc_dqblk.dqb_curspace; + curinodes += oqctl->qc_dqblk.dqb_curinodes; + } + } + oqctl->qc_dqblk.dqb_curspace = curspace; + oqctl->qc_dqblk.dqb_curinodes = curinodes; - RETURN(rc); + RETURN(rc); } static int lmv_merge_attr(struct obd_export *exp, diff --git a/lustre/lmv/lmv_qos.c b/lustre/lmv/lmv_qos.c index 1708c00..44b98bd 100644 --- a/lustre/lmv/lmv_qos.c +++ b/lustre/lmv/lmv_qos.c @@ -79,7 +79,6 @@ static int lmv_qos_calc_ppts(struct lmv_obd *lmv) __u64 ba_max, ba_min, ba; __u64 ia_max, ia_min, ia; __u32 num_active; - unsigned int i; int prio_wide; time64_t now, age; __u32 maxage = lmv->desc.ld_qos_maxage; @@ -115,9 +114,8 @@ static int lmv_qos_calc_ppts(struct lmv_obd *lmv) now = ktime_get_real_seconds(); /* Calculate server penalty per object */ - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) + lmv_foreach_tgt(lmv, tgt) { + if (!tgt->ltd_exp || !tgt->ltd_active) continue; /* bavail >> 16 to avoid overflow */ @@ -165,9 +163,8 @@ static int lmv_qos_calc_ppts(struct lmv_obd *lmv) * we have to double the MDT penalty */ num_active = 2; - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) + lmv_foreach_tgt(lmv, tgt) { + if (!tgt->ltd_exp || !tgt->ltd_active) continue; tgt->ltd_qos.ltq_penalty_per_obj <<= 1; @@ -266,7 +263,6 @@ static int lmv_qos_used(struct lmv_obd *lmv, struct lu_tgt_desc *tgt, { struct lu_tgt_qos *ltq; struct lu_svr_qos *svr; - unsigned int i; ENTRY; @@ -304,9 +300,8 @@ static int lmv_qos_used(struct lmv_obd *lmv, struct lu_tgt_desc *tgt, *total_wt = 0; /* Decrease all MDT penalties */ - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - ltq = &lmv->tgts[i]->ltd_qos; - if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) + lmv_foreach_tgt(lmv, tgt) { + if (!tgt->ltd_exp || !tgt->ltd_active) continue; if (ltq->ltq_penalty < ltq->ltq_penalty_per_obj) @@ -314,7 +309,7 @@ static int lmv_qos_used(struct lmv_obd *lmv, struct lu_tgt_desc *tgt, else ltq->ltq_penalty -= ltq->ltq_penalty_per_obj; - lmv_qos_calc_weight(lmv->tgts[i]); + lmv_qos_calc_weight(tgt); /* Recalc the total weight of usable osts */ if (ltq->ltq_usable) @@ -323,7 +318,7 @@ static int lmv_qos_used(struct lmv_obd *lmv, struct lu_tgt_desc *tgt, CDEBUG(D_OTHER, "recalc tgt %d usable=%d avail=%llu" " tgtppo=%llu tgtp=%llu svrppo=%llu" " svrp=%llu wt=%llu\n", - i, ltq->ltq_usable, + tgt->ltd_index, ltq->ltq_usable, tgt_statfs_bavail(tgt) >> 10, ltq->ltq_penalty_per_obj >> 10, ltq->ltq_penalty >> 10, @@ -341,7 +336,6 @@ struct lu_tgt_desc *lmv_locate_tgt_qos(struct lmv_obd *lmv, __u32 *mdt) __u64 total_weight = 0; __u64 cur_weight = 0; __u64 rand; - int i; int rc; ENTRY; @@ -358,11 +352,7 @@ struct lu_tgt_desc *lmv_locate_tgt_qos(struct lmv_obd *lmv, __u32 *mdt) if (rc) GOTO(unlock, tgt = ERR_PTR(rc)); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - if (!tgt) - continue; - + lmv_foreach_tgt(lmv, tgt) { tgt->ltd_qos.ltq_usable = 0; if (!tgt->ltd_exp || !tgt->ltd_active) continue; @@ -374,10 +364,8 @@ struct lu_tgt_desc *lmv_locate_tgt_qos(struct lmv_obd *lmv, __u32 *mdt) rand = lu_prandom_u64_max(total_weight); - for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[i]; - - if (!tgt || !tgt->ltd_qos.ltq_usable) + lmv_foreach_tgt(lmv, tgt) { + if (!tgt->ltd_qos.ltq_usable) continue; cur_weight += tgt->ltd_qos.ltq_weight; @@ -406,17 +394,18 @@ struct lu_tgt_desc *lmv_locate_tgt_rr(struct lmv_obd *lmv, __u32 *mdt) spin_lock(&lmv->lmv_qos.lq_rr.lqr_alloc); for (i = 0; i < lmv->desc.ld_tgt_count; i++) { - tgt = lmv->tgts[(i + lmv->lmv_qos_rr_index) % - lmv->desc.ld_tgt_count]; - if (tgt && tgt->ltd_exp && tgt->ltd_active) { - *mdt = tgt->ltd_index; - lmv->lmv_qos_rr_index = - (i + lmv->lmv_qos_rr_index + 1) % - lmv->desc.ld_tgt_count; - spin_unlock(&lmv->lmv_qos.lq_rr.lqr_alloc); - - RETURN(tgt); - } + tgt = lmv_tgt(lmv, + (i + lmv->lmv_qos_rr_index) % lmv->desc.ld_tgt_count); + if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) + continue; + + *mdt = tgt->ltd_index; + lmv->lmv_qos_rr_index = + (i + lmv->lmv_qos_rr_index + 1) % + lmv->desc.ld_tgt_count; + spin_unlock(&lmv->lmv_qos.lq_rr.lqr_alloc); + + RETURN(tgt); } spin_unlock(&lmv->lmv_qos.lq_rr.lqr_alloc); diff --git a/lustre/lmv/lproc_lmv.c b/lustre/lmv/lproc_lmv.c index 981b032..52f5953 100644 --- a/lustre/lmv/lproc_lmv.c +++ b/lustre/lmv/lproc_lmv.c @@ -181,16 +181,19 @@ LUSTRE_RW_ATTR(qos_threshold_rr); #ifdef CONFIG_PROC_FS static void *lmv_tgt_seq_start(struct seq_file *p, loff_t *pos) { - struct obd_device *dev = p->private; - struct lmv_obd *lmv = &dev->u.lmv; + struct obd_device *dev = p->private; + struct lmv_obd *lmv = &dev->u.lmv; + struct lu_tgt_desc *tgt; + + while (*pos < lmv->lmv_mdt_descs.ltd_tgts_size) { + tgt = lmv_tgt(lmv, (__u32)*pos); + if (tgt) + return tgt; - while (*pos < lmv->tgts_size) { - if (lmv->tgts[*pos]) - return lmv->tgts[*pos]; ++*pos; } - return NULL; + return NULL; } static void lmv_tgt_seq_stop(struct seq_file *p, void *v) @@ -199,17 +202,20 @@ static void lmv_tgt_seq_stop(struct seq_file *p, void *v) static void *lmv_tgt_seq_next(struct seq_file *p, void *v, loff_t *pos) { - struct obd_device *dev = p->private; - struct lmv_obd *lmv = &dev->u.lmv; + struct obd_device *dev = p->private; + struct lmv_obd *lmv = &dev->u.lmv; + struct lu_tgt_desc *tgt; ++*pos; - while (*pos < lmv->tgts_size) { - if (lmv->tgts[*pos]) - return lmv->tgts[*pos]; + while (*pos < lmv->lmv_mdt_descs.ltd_tgts_size) { + tgt = lmv_tgt(lmv, (__u32)*pos); + if (tgt) + return tgt; + ++*pos; } - return NULL; + return NULL; } static int lmv_tgt_seq_show(struct seq_file *p, void *v) diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 8d9c1ef..a452e5d 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -244,27 +244,21 @@ static int lod_sub_process_config(const struct lu_env *env, struct lod_tgt_descs *ltd, struct lustre_cfg *lcfg) { - struct lu_device *next; + struct lu_device *next; + struct lu_tgt_desc *tgt; int rc = 0; - unsigned int i; lod_getref(ltd); - if (ltd->ltd_tgts_size <= 0) { - lod_putref(lod, ltd); - return 0; - } - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - struct lod_tgt_desc *tgt; + ltd_foreach_tgt(ltd, tgt) { int rc1; - tgt = LTD_TGT(ltd, i); LASSERT(tgt && tgt->ltd_tgt); next = &tgt->ltd_tgt->dd_lu_dev; rc1 = next->ld_ops->ldo_process_config(env, next, lcfg); if (rc1) { CERROR("%s: error cleaning up LOD index %u: cmd %#x : rc = %d\n", - lod2obd(lod)->obd_name, i, lcfg->lcfg_command, - rc1); + lod2obd(lod)->obd_name, tgt->ltd_index, + lcfg->lcfg_command, rc1); rc = rc1; } } @@ -368,7 +362,6 @@ static int lod_sub_recovery_thread(void *arg) struct lod_tgt_desc *tgt = NULL; time64_t start; int retries = 0; - int i; int rc; ENTRY; @@ -459,8 +452,7 @@ again: GOTO(out, rc = 0); } - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { if (!tgt->ltd_got_update_log) { spin_unlock(&lod->lod_lock); GOTO(out, rc = 0); @@ -605,7 +597,7 @@ int lod_sub_init_llog(const struct lu_env *env, struct lod_device *lod, struct ptlrpc_thread *thread; struct task_struct *task; struct l_wait_info lwi = { 0 }; - struct lod_tgt_desc *sub_ltd = NULL; + struct lod_tgt_desc *subtgt = NULL; u32 index; u32 master_index; int rc; @@ -624,30 +616,27 @@ int lod_sub_init_llog(const struct lu_env *env, struct lod_device *lod, thread = &lod->lod_child_recovery_thread; index = master_index; } else { - struct lod_tgt_descs *ltd = &lod->lod_mdt_descs; - struct lod_tgt_desc *tgt = NULL; - unsigned int i; + struct lu_tgt_desc *tgt; - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(&lod->lod_mdt_descs, tgt) { if (tgt->ltd_tgt == dt) { index = tgt->ltd_index; - sub_ltd = tgt; + subtgt = tgt; break; } } - LASSERT(sub_ltd != NULL); - OBD_ALLOC_PTR(sub_ltd->ltd_recovery_thread); - if (!sub_ltd->ltd_recovery_thread) + LASSERT(subtgt != NULL); + OBD_ALLOC_PTR(subtgt->ltd_recovery_thread); + if (!subtgt->ltd_recovery_thread) GOTO(free_lrd, rc = -ENOMEM); - thread = sub_ltd->ltd_recovery_thread; + thread = subtgt->ltd_recovery_thread; } CDEBUG(D_INFO, "%s init sub log %s\n", lod2obd(lod)->obd_name, dt->dd_lu_dev.ld_obd->obd_name); lrd->lrd_lod = lod; - lrd->lrd_ltd = sub_ltd; + lrd->lrd_ltd = subtgt; lrd->lrd_thread = thread; lrd->lrd_idx = index; init_waitqueue_head(&thread->t_ctl_waitq); @@ -680,8 +669,8 @@ out_llog: lod_sub_fini_llog(env, dt, thread); free_thread: if (lod->lod_child != dt) { - OBD_FREE_PTR(sub_ltd->ltd_recovery_thread); - sub_ltd->ltd_recovery_thread = NULL; + OBD_FREE_PTR(subtgt->ltd_recovery_thread); + subtgt->ltd_recovery_thread = NULL; } free_lrd: OBD_FREE_PTR(lrd); @@ -701,7 +690,7 @@ static void lod_sub_stop_recovery_threads(const struct lu_env *env, { struct lod_tgt_descs *ltd = &lod->lod_mdt_descs; struct ptlrpc_thread *thread; - unsigned int i; + struct lu_tgt_desc *tgt; /* * Stop the update log commit cancel threads and finish master @@ -716,10 +705,7 @@ static void lod_sub_stop_recovery_threads(const struct lu_env *env, } lod_getref(ltd); - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - struct lod_tgt_desc *tgt; - - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { thread = tgt->ltd_recovery_thread; if (thread && thread->t_flags & SVC_RUNNING) { thread->t_flags = SVC_STOPPING; @@ -746,7 +732,7 @@ static void lod_sub_fini_all_llogs(const struct lu_env *env, struct lod_device *lod) { struct lod_tgt_descs *ltd = &lod->lod_mdt_descs; - unsigned int i; + struct lu_tgt_desc *tgt; /* * Stop the update log commit cancel threads and finish master @@ -755,14 +741,9 @@ static void lod_sub_fini_all_llogs(const struct lu_env *env, lod_sub_fini_llog(env, lod->lod_child, &lod->lod_child_recovery_thread); lod_getref(ltd); - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - struct lod_tgt_desc *tgt; - - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) lod_sub_fini_llog(env, tgt->ltd_tgt, tgt->ltd_recovery_thread); - } - lod_putref(lod, ltd); } @@ -802,10 +783,10 @@ static char *lod_show_update_logs_retrievers(void *data, int *size, int *count) (*count)++; } - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { if (!tgt->ltd_got_update_log) { - rc = snprintf(buf + len, *size - len, " %04x", i); + rc = snprintf(buf + len, *size - len, " %04x", + tgt->ltd_index); if (unlikely(rc <= 0)) break; @@ -994,9 +975,9 @@ static int lod_process_config(const struct lu_env *env, if (strstr(param, "osp") && strstr(param, ".active=")) { struct lod_tgt_descs *ltd = &lod->lod_mdt_descs; struct lod_tgt_desc *sub_tgt = NULL; + struct lu_tgt_desc *tgt; char *ptr; char *tmp; - int i; ptr = strstr(param, "."); *ptr = '\0'; @@ -1008,10 +989,7 @@ static int lod_process_config(const struct lu_env *env, GOTO(out, rc); } - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - struct lod_tgt_desc *tgt; - - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { if (tgt->ltd_tgt->dd_lu_dev.ld_obd == obd) { sub_tgt = tgt; break; @@ -1172,8 +1150,8 @@ static int lod_recovery_complete(const struct lu_env *env, static int lod_sub_init_llogs(const struct lu_env *env, struct lod_device *lod) { struct lod_tgt_descs *ltd = &lod->lod_mdt_descs; + struct lu_tgt_desc *tgt; int rc; - unsigned int i; ENTRY; @@ -1188,10 +1166,7 @@ static int lod_sub_init_llogs(const struct lu_env *env, struct lod_device *lod) if (rc < 0) RETURN(rc); - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - struct lod_tgt_desc *tgt; - - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { rc = lod_sub_init_llog(env, lod, tgt->ltd_tgt); if (rc != 0) break; @@ -1724,38 +1699,6 @@ out: } /** - * Allocate and initialize target table. - * - * A helper function to initialize the target table and allocate - * a bitmap of the available targets. - * - * \param[in] ltd target's table to initialize - * - * \retval 0 on success - * \retval negative negated errno on error - **/ -static int lod_tgt_desc_init(struct lod_tgt_descs *ltd) -{ - mutex_init(<d->ltd_mutex); - init_rwsem(<d->ltd_rw_sem); - - /* - * the OST array and bitmap are allocated/grown dynamically as OSTs are - * added to the LOD, see lod_add_device() - */ - ltd->ltd_tgt_bitmap = CFS_ALLOCATE_BITMAP(32); - if (!ltd->ltd_tgt_bitmap) - RETURN(-ENOMEM); - - ltd->ltd_tgts_size = 32; - ltd->ltd_tgtnr = 0; - - ltd->ltd_death_row = 0; - ltd->ltd_refcount = 0; - return 0; -} - -/** * Initialize LOD device at setup. * * Initializes the given LOD device using the original configuration command. @@ -1811,8 +1754,8 @@ static int lod_init0(const struct lu_env *env, struct lod_device *lod, spin_lock_init(&lod->lod_lock); spin_lock_init(&lod->lod_connects_lock); - lod_tgt_desc_init(&lod->lod_mdt_descs); - lod_tgt_desc_init(&lod->lod_ost_descs); + lu_tgt_descs_init(&lod->lod_mdt_descs); + lu_tgt_descs_init(&lod->lod_ost_descs); RETURN(0); diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 6cdc670..a7b4eed 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -70,38 +70,10 @@ struct pool_desc { #define pool_tgt_rw_sem(p) ((p)->pool_obds.op_rw_sem) #define lod_tgt_desc lu_tgt_desc - -#define TGT_PTRS 256 /* number of pointers at 1st level */ -#define TGT_PTRS_PER_BLOCK 256 /* number of pointers at 2nd level */ - -struct lod_tgt_desc_idx { - struct lod_tgt_desc *ldi_tgt[TGT_PTRS_PER_BLOCK]; -}; - -#define LTD_TGT(ltd, index) \ - ((ltd)->ltd_tgt_idx[(index) / \ - TGT_PTRS_PER_BLOCK]->ldi_tgt[(index) % TGT_PTRS_PER_BLOCK]) +#define lod_tgt_descs lu_tgt_descs #define OST_TGT(lod, index) LTD_TGT(&lod->lod_ost_descs, index) #define MDT_TGT(lod, index) LTD_TGT(&lod->lod_mdt_descs, index) -struct lod_tgt_descs { - /* list of known TGTs */ - struct lod_tgt_desc_idx *ltd_tgt_idx[TGT_PTRS]; - /* Size of the lod_tgts array, granted to be a power of 2 */ - __u32 ltd_tgts_size; - /* number of registered TGTs */ - __u32 ltd_tgtnr; - /* bitmap of TGTs available */ - struct cfs_bitmap *ltd_tgt_bitmap; - /* TGTs scheduled to be deleted */ - __u32 ltd_death_row; - /* Table refcount used for delayed deletion */ - int ltd_refcount; - /* mutex to serialize concurrent updates to the tgt table */ - struct mutex ltd_mutex; - /* read/write semaphore used for array relocation */ - struct rw_semaphore ltd_rw_sem; -}; struct lod_avoid_guide { /* ids of OSSs avoid guidance */ diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 8c3776e..3d04759 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -92,15 +92,13 @@ void lod_putref(struct lod_device *lod, struct lod_tgt_descs *ltd) continue; list_add(&tgt_desc->ltd_kill, &kill); - LTD_TGT(ltd, idx) = NULL; /*FIXME: only support ost pool for now */ if (ltd == &lod->lod_ost_descs) { lod_ost_pool_remove(&lod->lod_pool_info, idx); if (tgt_desc->ltd_active) lod->lod_desc.ld_active_tgt_count--; } - ltd->ltd_tgtnr--; - cfs_bitmap_clear(ltd->ltd_tgt_bitmap, idx); + lu_tgt_descs_del(ltd, tgt_desc); ltd->ltd_death_row--; } mutex_unlock(<d->ltd_mutex); @@ -133,60 +131,6 @@ void lod_putref(struct lod_device *lod, struct lod_tgt_descs *ltd) } /** - * Expand size of target table. - * - * When the target table is full, we have to extend the table. To do so, - * we allocate new memory with some reserve, move data from the old table - * to the new one and release memory consumed by the old table. - * Notice we take ltd_rw_sem exclusively to ensure atomic switch. - * - * \param[in] ltd target table - * \param[in] newsize new size of the table - * - * \retval 0 on success - * \retval -ENOMEM if reallocation failed - */ -static int ltd_bitmap_resize(struct lod_tgt_descs *ltd, __u32 newsize) -{ - struct cfs_bitmap *new_bitmap, *old_bitmap = NULL; - int rc = 0; - ENTRY; - - /* grab write reference on the lod. Relocating the array requires - * exclusive access */ - - down_write(<d->ltd_rw_sem); - if (newsize <= ltd->ltd_tgts_size) - /* someone else has already resize the array */ - GOTO(out, rc = 0); - - /* allocate new bitmap */ - new_bitmap = CFS_ALLOCATE_BITMAP(newsize); - if (!new_bitmap) - GOTO(out, rc = -ENOMEM); - - if (ltd->ltd_tgts_size > 0) { - /* the bitmap already exists, we need - * to copy data from old one */ - cfs_bitmap_copy(new_bitmap, ltd->ltd_tgt_bitmap); - old_bitmap = ltd->ltd_tgt_bitmap; - } - - ltd->ltd_tgts_size = newsize; - ltd->ltd_tgt_bitmap = new_bitmap; - - if (old_bitmap) - CFS_FREE_BITMAP(old_bitmap); - - CDEBUG(D_CONFIG, "tgt size: %d\n", ltd->ltd_tgts_size); - - EXIT; -out: - up_write(<d->ltd_rw_sem); - return rc; -} - -/** * Connect LOD to a new OSP and add it to the target table. * * Connect to the OSP device passed, initialize all the internal @@ -219,7 +163,6 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, struct lustre_cfg *lcfg; struct obd_uuid obd_uuid; bool for_ost; - bool lock = false; bool connected = false; ENTRY; @@ -317,43 +260,9 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, tgt_desc->ltd_index = index; tgt_desc->ltd_active = active; - lod_getref(ltd); - if (index >= ltd->ltd_tgts_size) { - /* we have to increase the size of the lod_osts array */ - __u32 newsize; - - newsize = max(ltd->ltd_tgts_size, (__u32)2); - while (newsize < index + 1) - newsize = newsize << 1; - - /* lod_bitmap_resize() needs lod_rw_sem - * which we hold with th reference */ - lod_putref(lod, ltd); - - rc = ltd_bitmap_resize(ltd, newsize); - if (rc) - GOTO(out_desc, rc); - - lod_getref(ltd); - } - + down_write(<d->ltd_rw_sem); mutex_lock(<d->ltd_mutex); - lock = true; - if (cfs_bitmap_check(ltd->ltd_tgt_bitmap, index)) { - CERROR("%s: device %d is registered already\n", obd->obd_name, - index); - GOTO(out_mutex, rc = -EEXIST); - } - - if (ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK] == NULL) { - OBD_ALLOC_PTR(ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK]); - if (ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK] == NULL) { - CERROR("can't allocate index to add %s\n", - obd->obd_name); - GOTO(out_mutex, rc = -ENOMEM); - } - } - + lu_tgt_descs_add(ltd, tgt_desc); if (for_ost) { /* pool and qos are not supported for MDS stack yet */ rc = lod_ost_pool_add(&lod->lod_pool_info, index, @@ -377,13 +286,9 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, if (active) lod->lod_desc.ld_active_tgt_count++; } - - LTD_TGT(ltd, index) = tgt_desc; - cfs_bitmap_set(ltd->ltd_tgt_bitmap, index); - ltd->ltd_tgtnr++; mutex_unlock(<d->ltd_mutex); - lod_putref(lod, ltd); - lock = false; + up_write(<d->ltd_rw_sem); + if (lod->lod_recovery_completed) lu_dev->ld_ops->ldo_recovery_complete(env, lu_dev); @@ -407,26 +312,20 @@ out_fini_llog: lod_sub_fini_llog(env, tgt_desc->ltd_tgt, tgt_desc->ltd_recovery_thread); out_ltd: - lod_getref(ltd); + down_write(<d->ltd_rw_sem); mutex_lock(<d->ltd_mutex); - lock = true; if (!for_ost && LTD_TGT(ltd, index)->ltd_recovery_thread != NULL) { struct ptlrpc_thread *thread; thread = LTD_TGT(ltd, index)->ltd_recovery_thread; OBD_FREE_PTR(thread); } - ltd->ltd_tgtnr--; - cfs_bitmap_clear(ltd->ltd_tgt_bitmap, index); - LTD_TGT(ltd, index) = NULL; out_pool: lod_ost_pool_remove(&lod->lod_pool_info, index); out_mutex: - if (lock) { - mutex_unlock(<d->ltd_mutex); - lod_putref(lod, ltd); - } -out_desc: + lu_tgt_descs_del(ltd, tgt_desc); + mutex_unlock(<d->ltd_mutex); + up_write(<d->ltd_rw_sem); OBD_FREE_PTR(tgt_desc); out_cleanup: /* XXX OSP needs us to send down LCFG_CLEANUP because it uses @@ -497,18 +396,16 @@ int lod_fini_tgt(const struct lu_env *env, struct lod_device *lod, if (ltd->ltd_tgts_size <= 0) return 0; + lod_getref(ltd); mutex_lock(<d->ltd_mutex); cfs_foreach_bit(ltd->ltd_tgt_bitmap, idx) __lod_del_device(env, lod, ltd, idx, for_ost); mutex_unlock(<d->ltd_mutex); lod_putref(lod, ltd); - CFS_FREE_BITMAP(ltd->ltd_tgt_bitmap); - for (idx = 0; idx < TGT_PTRS; idx++) { - if (ltd->ltd_tgt_idx[idx]) - OBD_FREE_PTR(ltd->ltd_tgt_idx[idx]); - } - ltd->ltd_tgts_size = 0; + + lu_tgt_descs_fini(ltd); + return 0; } diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index fb8ea42..71b0fa1 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -5607,12 +5607,10 @@ static int lod_declare_create(const struct lu_env *env, struct dt_object *dt, struct lod_tgt_descs *ltd; struct lod_tgt_desc *tgt = NULL; bool found_mdt = false; - int i; lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); ltd = &lod->lod_mdt_descs; - cfs_foreach_bit(ltd->ltd_tgt_bitmap, i) { - tgt = LTD_TGT(ltd, i); + ltd_foreach_tgt(ltd, tgt) { if (tgt->ltd_index == lo->ldo_dir_stripe_offset) { found_mdt = true; diff --git a/lustre/obdclass/Makefile.in b/lustre/obdclass/Makefile.in index 021d0e1..128ecda 100644 --- a/lustre/obdclass/Makefile.in +++ b/lustre/obdclass/Makefile.in @@ -13,6 +13,7 @@ obdclass-all-objs += linkea.o obdclass-all-objs += kernelcomm.o jobid.o obdclass-all-objs += integrity.o obd_cksum.o obdclass-all-objs += lu_qos.o +obdclass-all-objs += lu_tgt_descs.o @SERVER_TRUE@obdclass-all-objs += acl.o @SERVER_TRUE@obdclass-all-objs += idmap.o diff --git a/lustre/obdclass/lu_tgt_descs.c b/lustre/obdclass/lu_tgt_descs.c new file mode 100644 index 0000000..cc5ab2c --- /dev/null +++ b/lustre/obdclass/lu_tgt_descs.c @@ -0,0 +1,197 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.gnu.org/licenses/gpl-2.0.html + * + * GPL HEADER END + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * + * lustre/obdclass/lu_tgt_descs.c + * + * Lustre target descriptions + * These are the only exported functions, they provide some generic + * infrastructure for target description management used by LOD/LMV + * + */ + +#define DEBUG_SUBSYSTEM S_CLASS + +#include +#include +#include +#include /* hash_long() */ +#include +#include +#include +#include +#include +#include + +/** + * Allocate and initialize target table. + * + * A helper function to initialize the target table and allocate + * a bitmap of the available targets. + * + * \param[in] ltd target's table to initialize + * + * \retval 0 on success + * \retval negative negated errno on error + **/ +int lu_tgt_descs_init(struct lu_tgt_descs *ltd) +{ + mutex_init(<d->ltd_mutex); + init_rwsem(<d->ltd_rw_sem); + + /* + * the tgt array and bitmap are allocated/grown dynamically as tgts are + * added to the LOD/LMV, see lu_tgt_descs_add() + */ + ltd->ltd_tgt_bitmap = CFS_ALLOCATE_BITMAP(BITS_PER_LONG); + if (!ltd->ltd_tgt_bitmap) + return -ENOMEM; + + ltd->ltd_tgts_size = BITS_PER_LONG; + ltd->ltd_tgtnr = 0; + + ltd->ltd_death_row = 0; + ltd->ltd_refcount = 0; + + return 0; +} +EXPORT_SYMBOL(lu_tgt_descs_init); + +/** + * Free bitmap and target table pages. + * + * \param[in] ltd target table + */ +void lu_tgt_descs_fini(struct lu_tgt_descs *ltd) +{ + int i; + + CFS_FREE_BITMAP(ltd->ltd_tgt_bitmap); + for (i = 0; i < TGT_PTRS; i++) { + if (ltd->ltd_tgt_idx[i]) + OBD_FREE_PTR(ltd->ltd_tgt_idx[i]); + } + ltd->ltd_tgts_size = 0; +} +EXPORT_SYMBOL(lu_tgt_descs_fini); + +/** + * Expand size of target table. + * + * When the target table is full, we have to extend the table. To do so, + * we allocate new memory with some reserve, move data from the old table + * to the new one and release memory consumed by the old table. + * + * \param[in] ltd target table + * \param[in] newsize new size of the table + * + * \retval 0 on success + * \retval -ENOMEM if reallocation failed + */ +static int lu_tgt_descs_resize(struct lu_tgt_descs *ltd, __u32 newsize) +{ + struct cfs_bitmap *new_bitmap, *old_bitmap = NULL; + + /* someone else has already resize the array */ + if (newsize <= ltd->ltd_tgts_size) + return 0; + + new_bitmap = CFS_ALLOCATE_BITMAP(newsize); + if (!new_bitmap) + return -ENOMEM; + + if (ltd->ltd_tgts_size > 0) { + /* the bitmap already exists, copy data from old one */ + cfs_bitmap_copy(new_bitmap, ltd->ltd_tgt_bitmap); + old_bitmap = ltd->ltd_tgt_bitmap; + } + + ltd->ltd_tgts_size = newsize; + ltd->ltd_tgt_bitmap = new_bitmap; + + if (old_bitmap) + CFS_FREE_BITMAP(old_bitmap); + + CDEBUG(D_CONFIG, "tgt size: %d\n", ltd->ltd_tgts_size); + + return 0; +} + +/** + * Add new target to target table. + * + * Extend target table if it's full, update target table and bitmap. + * Notice we need to take ltd_rw_sem exclusively before entry to ensure + * atomic switch. + * + * \param[in] ltd target table + * \param[in] tgt new target desc + * + * \retval 0 on success + * \retval -ENOMEM if reallocation failed + * -EEXIST if target existed + */ +int lu_tgt_descs_add(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt) +{ + __u32 index = tgt->ltd_index; + int rc; + + ENTRY; + + if (index >= ltd->ltd_tgts_size) { + __u32 newsize = 1; + + while (newsize < index + 1) + newsize = newsize << 1; + + rc = lu_tgt_descs_resize(ltd, newsize); + if (rc) + RETURN(rc); + } else if (cfs_bitmap_check(ltd->ltd_tgt_bitmap, index)) { + RETURN(-EEXIST); + } + + if (ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK] == NULL) { + OBD_ALLOC_PTR(ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK]); + if (ltd->ltd_tgt_idx[index / TGT_PTRS_PER_BLOCK] == NULL) + RETURN(-ENOMEM); + } + + LTD_TGT(ltd, tgt->ltd_index) = tgt; + cfs_bitmap_set(ltd->ltd_tgt_bitmap, tgt->ltd_index); + ltd->ltd_tgtnr++; + + RETURN(0); +} +EXPORT_SYMBOL(lu_tgt_descs_add); + +/** + * Delete target from target table + */ +void lu_tgt_descs_del(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt) +{ + LTD_TGT(ltd, tgt->ltd_index) = NULL; + cfs_bitmap_clear(ltd->ltd_tgt_bitmap, tgt->ltd_index); + ltd->ltd_tgtnr--; +} +EXPORT_SYMBOL(lu_tgt_descs_del);