X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Flod%2Flod_qos.c;h=f4a77835f0d38ee71b19c8b8a0f3dd545f9e6971;hp=a027947a357b0d5fb8081d05c111f0fc4174015f;hb=dd15646cc557357af7f8751dd136293b12588ce4;hpb=6dc37759cfb22727ac5d776c38b72e8638563fd8 diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index a027947..f4a7783 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -65,17 +65,17 @@ static inline int lod_statfs_check(struct lu_tgt_descs *ltd, { struct obd_statfs *sfs = &tgt->ltd_statfs; - if (((sfs->os_state & OS_STATE_ENOSPC) || - (!ltd->ltd_is_mdt && sfs->os_state & OS_STATE_ENOINO && + if (((sfs->os_state & OS_STATFS_ENOSPC) || + (!ltd->ltd_is_mdt && sfs->os_state & OS_STATFS_ENOINO && sfs->os_fprecreated == 0))) return -ENOSPC; /* If the OST is readonly then we can't allocate objects there */ - if (sfs->os_state & OS_STATE_READONLY) + if (sfs->os_state & OS_STATFS_READONLY) return -EROFS; /* object precreation is skipped on the OST with max_create_count=0 */ - if (!ltd->ltd_is_mdt && sfs->os_state & OS_STATE_NOPRECREATE) + if (!ltd->ltd_is_mdt && sfs->os_state & OS_STATFS_NOPRECREATE) return -ENOBUFS; return 0; @@ -100,15 +100,18 @@ static inline int lod_statfs_check(struct lu_tgt_descs *ltd, */ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d, struct lu_tgt_descs *ltd, - struct lu_tgt_desc *tgt) + struct lu_tgt_desc *tgt, __u64 reserve) { + struct obd_statfs_info info = { 0 }; struct lov_desc *desc = <d->ltd_lov_desc; int rc; + ENTRY; LASSERT(d); LASSERT(tgt); - rc = dt_statfs(env, tgt->ltd_tgt, &tgt->ltd_statfs); + info.os_enable_pre = 1; + rc = dt_statfs_info(env, tgt->ltd_tgt, &tgt->ltd_statfs, &info); if (rc && rc != -ENOTCONN) CERROR("%s: statfs: rc = %d\n", lod2obd(d)->obd_name, rc); @@ -118,6 +121,11 @@ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d, return rc; } + if (reserve && + (reserve + (info.os_reserved_mb_low << 20) > + tgt->ltd_statfs.os_bavail * tgt->ltd_statfs.os_bsize)) + return -ENOSPC; + /* check whether device has changed state (active, inactive) */ if (rc != 0 && tgt->ltd_active) { /* turned inactive? */ @@ -129,8 +137,8 @@ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d, LASSERT(desc->ld_active_tgt_count > 0); desc->ld_active_tgt_count--; - ltd->ltd_qos.lq_dirty = 1; - ltd->ltd_qos.lq_rr.lqr_dirty = 1; + set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags); + set_bit(LQ_DIRTY, <d->ltd_qos.lq_rr.lqr_flags); CDEBUG(D_CONFIG, "%s: turns inactive\n", tgt->ltd_exp->exp_obd->obd_name); } @@ -145,15 +153,22 @@ static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d, tgt->ltd_active = 1; tgt->ltd_connecting = 0; desc->ld_active_tgt_count++; - ltd->ltd_qos.lq_dirty = 1; - ltd->ltd_qos.lq_rr.lqr_dirty = 1; + set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags); + set_bit(LQ_DIRTY, <d->ltd_qos.lq_rr.lqr_flags); CDEBUG(D_CONFIG, "%s: turns active\n", tgt->ltd_exp->exp_obd->obd_name); } spin_unlock(&d->lod_lock); } + if (rc == -ENOTCONN) { + /* In case that the ENOTCONN for inactive OST state is + * mistreated as MDT disconnection state by the client, + * this error should be changed to someone else. + */ + rc = -EREMOTEIO; + } - return rc; + RETURN(rc); } static int lod_is_tgt_usable(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt) @@ -203,12 +218,12 @@ void lod_qos_statfs_update(const struct lu_env *env, struct lod_device *lod, ltd_foreach_tgt(ltd, tgt) { avail = tgt->ltd_statfs.os_bavail; - if (lod_statfs_and_check(env, lod, ltd, tgt)) + if (lod_statfs_and_check(env, lod, ltd, tgt, 0)) continue; if (tgt->ltd_statfs.os_bavail != avail) /* recalculate weigths */ - ltd->ltd_qos.lq_dirty = 1; + set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags); } obd->obd_osfs_age = ktime_get_seconds(); @@ -247,7 +262,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, int rc; ENTRY; - if (!lqr->lqr_dirty) { + if (!test_bit(LQ_DIRTY, &lqr->lqr_flags)) { LASSERT(lqr->lqr_pool.op_size); RETURN(0); } @@ -259,7 +274,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, * Check again. While we were sleeping on @lq_rw_sem something could * change. */ - if (!lqr->lqr_dirty) { + if (!test_bit(LQ_DIRTY, &lqr->lqr_flags)) { LASSERT(lqr->lqr_pool.op_size); up_write(<d->ltd_qos.lq_rw_sem); RETURN(0); @@ -272,7 +287,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, deleting from the pool. The lq_rw_sem insures that nobody else is reading. */ lqr->lqr_pool.op_count = real_count; - rc = lod_tgt_pool_extend(&lqr->lqr_pool, real_count); + rc = tgt_pool_extend(&lqr->lqr_pool, real_count); if (rc) { up_write(<d->ltd_qos.lq_rw_sem); RETURN(rc); @@ -288,8 +303,8 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, for (i = 0; i < lqr->lqr_pool.op_count; i++) { int next; - if (!cfs_bitmap_check(ltd->ltd_tgt_bitmap, - src_pool->op_array[i])) + if (!test_bit(src_pool->op_array[i], + ltd->ltd_tgt_bitmap)) continue; tgt = LTD_TGT(ltd, src_pool->op_array[i]); @@ -308,7 +323,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, } } - lqr->lqr_dirty = 0; + clear_bit(LQ_DIRTY, &lqr->lqr_flags); up_write(<d->ltd_qos.lq_rw_sem); if (placed != real_count) { @@ -320,7 +335,7 @@ static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd, LCONSOLE(D_WARNING, "rr #%d tgt idx=%d\n", i, lqr->lqr_pool.op_array[i]); } - lqr->lqr_dirty = 1; + set_bit(LQ_DIRTY, &lqr->lqr_flags); RETURN(-EAGAIN); } @@ -379,7 +394,7 @@ static struct dt_object *lod_qos_declare_object_on(const struct lu_env *env, */ o = lu_object_anon(env, nd, NULL); if (IS_ERR(o)) - GOTO(out, dt = ERR_PTR(PTR_ERR(o))); + GOTO(out, dt = ERR_CAST(o)); n = lu_object_locate(o->lo_header, nd->ld_type); if (unlikely(n == NULL)) { @@ -442,7 +457,7 @@ static inline int lod_qos_tgt_in_use_clear(const struct lu_env *env, if (info->lti_ea_store_size < sizeof(int) * stripes) lod_ea_store_resize(info, stripes * sizeof(int)); if (info->lti_ea_store_size < sizeof(int) * stripes) { - CERROR("can't allocate memory for ost-in-use array\n"); + CERROR("can't allocate memory for tgt-in-use array\n"); return -ENOMEM; } memset(info->lti_ea_store, -1, sizeof(int) * stripes); @@ -564,7 +579,7 @@ static inline bool lod_should_avoid_ost(struct lod_object *lo, bool used = false; int i; - if (!cfs_bitmap_check(lod->lod_ost_bitmap, index)) { + if (!test_bit(index, lod->lod_ost_bitmap)) { QOS_DEBUG("OST%d: been used in conflicting mirror component\n", index); return true; @@ -592,7 +607,7 @@ static inline bool lod_should_avoid_ost(struct lod_object *lo, return false; /* if the OSS has been used, check whether the OST has been used */ - if (!cfs_bitmap_check(lag->lag_ost_avoid_bitmap, index)) + if (!test_bit(index, lag->lag_ost_avoid_bitmap)) used = false; else QOS_DEBUG("OST%d: been used in conflicting mirror component\n", @@ -607,7 +622,8 @@ static int lod_check_and_reserve_ost(const struct lu_env *env, struct dt_object **stripe, __u32 *ost_indices, struct thandle *th, - bool *overstriped) + bool *overstriped, + __u64 reserve) { struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lod_avoid_guide *lag = &lod_env_info(env)->lti_avoid; @@ -618,7 +634,7 @@ static int lod_check_and_reserve_ost(const struct lu_env *env, ENTRY; - rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost); + rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost, reserve); if (rc) RETURN(rc); @@ -634,7 +650,7 @@ static int lod_check_and_reserve_ost(const struct lu_env *env, /* * try to use another OSP if this one is degraded */ - if (ost->ltd_statfs.os_state & OS_STATE_DEGRADED && speed < 2) { + if (ost->ltd_statfs.os_state & OS_STATFS_DEGRADED && speed < 2) { QOS_DEBUG("#%d: degraded\n", ost_idx); RETURN(rc); } @@ -717,7 +733,8 @@ static int lod_check_and_reserve_ost(const struct lu_env *env, */ static int lod_ost_alloc_rr(const struct lu_env *env, struct lod_object *lo, struct dt_object **stripe, __u32 *ost_indices, - int flags, struct thandle *th, int comp_idx) + int flags, struct thandle *th, int comp_idx, + __u64 reserve) { struct lod_layout_component *lod_comp; struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); @@ -800,7 +817,7 @@ repeat_find: stripe_idx, array_idx, ost_idx); if ((ost_idx == LOV_QOS_EMPTY) || - !cfs_bitmap_check(m->lod_ost_bitmap, ost_idx)) + !test_bit(ost_idx, m->lod_ost_bitmap)) continue; /* Fail Check before osc_precreate() is called @@ -811,7 +828,8 @@ repeat_find: spin_unlock(&lqr->lqr_alloc); rc = lod_check_and_reserve_ost(env, lo, lod_comp, ost_idx, speed, &stripe_idx, stripe, - ost_indices, th, &overstriped); + ost_indices, th, &overstriped, + reserve); spin_lock(&lqr->lqr_alloc); if (rc != 0 && OST_TGT(m, ost_idx)->ltd_connecting) @@ -857,6 +875,44 @@ out: RETURN(rc); } +static int +lod_qos_mdt_in_use_init(const struct lu_env *env, + const struct lu_tgt_descs *ltd, + u32 stripe_idx, u32 stripe_count, + const struct lu_tgt_pool *pool, + struct dt_object **stripes) +{ + u32 mdt_idx; + struct lu_tgt_desc *mdt; + int i, j; + int rc; + + rc = lod_qos_tgt_in_use_clear(env, stripe_count); + if (rc) + return rc; + + /* if stripe_idx > 1, we are splitting directory, mark existing stripes + * in_use. Because for either split or creation, stripe 0 is local, + * don't mark it in use. + */ + for (i = 1; i < stripe_idx; i++) { + LASSERT(stripes[i]); + for (j = 0; j < pool->op_count; j++) { + mdt_idx = pool->op_array[j]; + + if (!test_bit(mdt_idx, ltd->ltd_tgt_bitmap)) + continue; + + mdt = LTD_TGT(ltd, mdt_idx); + if (&mdt->ltd_tgt->dd_lu_dev == + stripes[i]->do_lu.lo_dev) + lod_qos_tgt_in_use(env, i, mdt_idx); + } + } + + return 0; +} + /** * Allocate a striping using round-robin algorithm. * @@ -870,7 +926,7 @@ out: * * \param[in] env execution environment for this thread * \param[in] lo LOD object - * \param[out] stripe striping created + * \param[out] stripes striping created * * \retval positive stripe objects allocated, including the first stripe * allocated outside @@ -878,7 +934,8 @@ out: * \retval negative negated errno for other failures */ int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo, - struct dt_object **stripe) + struct dt_object **stripes, u32 stripe_idx, + u32 stripe_count) { struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lu_tgt_descs *ltd = &lod->lod_mdt_descs; @@ -890,9 +947,8 @@ int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo, struct dt_object *dto; unsigned int pool_idx; unsigned int i; - u32 start_idx_temp; - u32 stripe_count = lo->ldo_dir_stripe_count; - u32 stripe_idx = 1; + u32 saved_idx = stripe_idx; + u32 start_mdt; u32 mdt_idx; bool use_degraded = false; int tgt_connecting = 0; @@ -906,7 +962,8 @@ int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo, if (rc) RETURN(rc); - rc = lod_qos_tgt_in_use_clear(env, stripe_count); + rc = lod_qos_mdt_in_use_init(env, ltd, stripe_idx, stripe_count, pool, + stripes); if (rc) RETURN(rc); @@ -927,10 +984,10 @@ int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo, (pool->op_count % (stripe_count - 1)) != 1) ++lqr->lqr_offset_idx; } - start_idx_temp = lqr->lqr_start_idx; + start_mdt = lqr->lqr_start_idx; repeat_find: - QOS_DEBUG("want %d start_idx %d start_count %d offset %d active %d count %d\n", + QOS_DEBUG("want=%d start_idx=%d start_count=%d offset=%d active=%d count=%d\n", stripe_count - 1, lqr->lqr_start_idx, lqr->lqr_start_count, lqr->lqr_offset_idx, pool->op_count, pool->op_count); @@ -946,7 +1003,7 @@ repeat_find: stripe_idx, pool_idx, mdt_idx); if (mdt_idx == LOV_QOS_EMPTY || - !cfs_bitmap_check(ltd->ltd_tgt_bitmap, mdt_idx)) + !test_bit(mdt_idx, ltd->ltd_tgt_bitmap)) continue; /* do not put >1 objects on one MDT */ @@ -961,15 +1018,15 @@ repeat_find: } /* try to use another OSP if this one is degraded */ - if (mdt->ltd_statfs.os_state & OS_STATE_DEGRADED && + if (mdt->ltd_statfs.os_state & OS_STATFS_DEGRADED && !use_degraded) { QOS_DEBUG("#%d: degraded\n", mdt_idx); continue; } spin_unlock(&lqr->lqr_alloc); - rc = obd_fid_alloc(env, mdt->ltd_exp, &fid, NULL); - if (rc) { + rc = dt_fid_alloc(env, mdt->ltd_tgt, &fid, NULL, NULL); + if (rc < 0) { QOS_DEBUG("#%d: alloc FID failed: %dl\n", mdt_idx, rc); spin_lock(&lqr->lqr_alloc); continue; @@ -990,14 +1047,13 @@ repeat_find: } lod_qos_tgt_in_use(env, stripe_idx, mdt_idx); - stripe[stripe_idx] = dto; - stripe_idx++; + stripes[stripe_idx++] = dto; } if (!use_degraded && stripe_idx < stripe_count) { - /* Try again, allowing slower OSCs */ + /* Try again, allowing slower MDTs */ use_degraded = true; - lqr->lqr_start_idx = start_idx_temp; + lqr->lqr_start_idx = start_mdt; tgt_connecting = 0; goto repeat_find; @@ -1005,7 +1061,7 @@ repeat_find: spin_unlock(&lqr->lqr_alloc); up_read(<d->ltd_qos.lq_rw_sem); - if (stripe_idx > 1) + if (stripe_idx > saved_idx) /* at least one stripe is allocated */ RETURN(stripe_idx); @@ -1043,7 +1099,7 @@ repeat_find: */ static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo, struct dt_object **stripe, __u32 *ost_indices, - struct thandle *th, int comp_idx) + struct thandle *th, int comp_idx, __u64 reserve) { struct lod_layout_component *lod_comp; struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); @@ -1086,7 +1142,7 @@ static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo, i++, array_idx = (array_idx + 1) % lod_comp->llc_stripe_count) { __u32 ost_idx = lod_comp->llc_ostlist.op_array[array_idx]; - if (!cfs_bitmap_check(m->lod_ost_bitmap, ost_idx)) { + if (!test_bit(ost_idx, m->lod_ost_bitmap)) { rc = -ENODEV; break; } @@ -1101,7 +1157,8 @@ static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo, } rc = lod_statfs_and_check(env, m, &m->lod_ost_descs, - LTD_TGT(&m->lod_ost_descs, ost_idx)); + LTD_TGT(&m->lod_ost_descs, ost_idx), + reserve); if (rc < 0) /* this OSP doesn't feel well */ break; @@ -1155,7 +1212,8 @@ static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo, static int lod_ost_alloc_specific(const struct lu_env *env, struct lod_object *lo, struct dt_object **stripe, __u32 *ost_indices, - int flags, struct thandle *th, int comp_idx) + int flags, struct thandle *th, int comp_idx, + __u64 reserve) { struct lod_layout_component *lod_comp; struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); @@ -1214,7 +1272,7 @@ repeat_find: i++, array_idx = (array_idx + 1) % ost_count) { ost_idx = osts->op_array[array_idx]; - if (!cfs_bitmap_check(m->lod_ost_bitmap, ost_idx)) + if (!test_bit(ost_idx, m->lod_ost_bitmap)) continue; /* Fail Check before osc_precreate() is called @@ -1248,7 +1306,8 @@ repeat_find: * start OST, then it can be skipped, otherwise skip it only * if it is inactive/recovering/out-of-space." */ - rc = lod_statfs_and_check(env, m, &m->lod_ost_descs, tgt); + rc = lod_statfs_and_check(env, m, &m->lod_ost_descs, + tgt, reserve); if (rc) { /* this OSP doesn't feel well */ continue; @@ -1349,7 +1408,8 @@ out: */ static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo, struct dt_object **stripe, __u32 *ost_indices, - int flags, struct thandle *th, int comp_idx) + int flags, struct thandle *th, int comp_idx, + __u64 reserve) { struct lod_layout_component *lod_comp; struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); @@ -1412,19 +1472,20 @@ static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo, good_osts = 0; /* Find all the OSTs that are valid stripe candidates */ for (i = 0; i < osts->op_count; i++) { - if (!cfs_bitmap_check(lod->lod_ost_bitmap, osts->op_array[i])) + if (!test_bit(osts->op_array[i], lod->lod_ost_bitmap)) continue; ost = OST_TGT(lod, osts->op_array[i]); ost->ltd_qos.ltq_usable = 0; - rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost); + rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, + ost, reserve); if (rc) { /* this OSP doesn't feel well */ continue; } - if (ost->ltd_statfs.os_state & OS_STATE_DEGRADED) + if (ost->ltd_statfs.os_state & OS_STATFS_DEGRADED) continue; /* Fail Check before osc_precreate() is called @@ -1542,9 +1603,8 @@ static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo, } /* makes sense to rebalance next time */ - lod->lod_ost_descs.ltd_qos.lq_dirty = 1; - lod->lod_ost_descs.ltd_qos.lq_same_space = 0; - + set_bit(LQ_DIRTY, &lod->lod_ost_descs.ltd_qos.lq_flags); + clear_bit(LQ_SAME_SPACE, &lod->lod_ost_descs.ltd_qos.lq_flags); rc = -EAGAIN; } @@ -1590,6 +1650,9 @@ out_nolock: * * \param[in] env execution environment for this thread * \param[in] lo LOD object + * \param[in] stripe_idx starting stripe index to allocate, if it's not + * 0, we are restriping directory + * \param[in] stripe_count total stripe count * \param[out] stripes striping created * * \retval positive stripes allocated, and it should be equal to @@ -1599,7 +1662,8 @@ out_nolock: * \retval negative errno on failure */ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, - struct dt_object **stripes) + struct dt_object **stripes, u32 stripe_idx, + u32 stripe_count) { struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lu_tgt_descs *ltd = &lod->lod_mdt_descs; @@ -1609,23 +1673,32 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, struct lu_tgt_desc *mdt; struct dt_object *dto; u64 total_weight = 0; - u32 stripe_count = lo->ldo_dir_stripe_count; - unsigned int nfound; + u32 saved_idx = stripe_idx; + u32 mdt_idx; unsigned int good_mdts; unsigned int i; int rc = 0; ENTRY; - if (stripe_count == 1) - RETURN(1); + LASSERT(stripe_idx <= stripe_count); + if (stripe_idx == stripe_count) + RETURN(stripe_count); + /* use MDT pool in @ltd, once MDT pool is supported in the future, it + * can be passed in as argument like OST object allocation. + */ pool = <d->ltd_tgt_pool; /* Detect -EAGAIN early, before expensive lock is taken. */ if (!ltd_qos_is_usable(ltd)) RETURN(-EAGAIN); + rc = lod_qos_mdt_in_use_init(env, ltd, stripe_idx, stripe_count, pool, + stripes); + if (rc) + RETURN(rc); + /* Do actual allocation, use write lock here. */ down_write(<d->ltd_qos.lq_rw_sem); @@ -1640,14 +1713,10 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, if (rc) GOTO(unlock, rc); - rc = lod_qos_tgt_in_use_clear(env, stripe_count); - if (rc) - GOTO(unlock, rc); - good_mdts = 0; - /* Find all the tgts that are valid stripe candidates */ + /* Find all the MDTs that are valid stripe candidates */ for (i = 0; i < pool->op_count; i++) { - if (!cfs_bitmap_check(ltd->ltd_tgt_bitmap, pool->op_array[i])) + if (!test_bit(pool->op_array[i], ltd->ltd_tgt_bitmap)) continue; mdt = LTD_TGT(ltd, pool->op_array[i]); @@ -1657,7 +1726,7 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, if (rc) continue; - if (mdt->ltd_statfs.os_state & OS_STATE_DEGRADED) + if (mdt->ltd_statfs.os_state & OS_STATFS_DEGRADED) continue; mdt->ltd_qos.ltq_usable = 1; @@ -1667,14 +1736,13 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, good_mdts++; } - QOS_DEBUG("found %d good tgts\n", good_mdts); + QOS_DEBUG("found %d good MDTs\n", good_mdts); - if (good_mdts < stripe_count - 1) + if (good_mdts < stripe_count - stripe_idx) GOTO(unlock, rc = -EAGAIN); - /* Find enough tgts with weighted random allocation. */ - nfound = 1; - while (nfound < stripe_count) { + /* Find enough MDTs with weighted random allocation. */ + while (stripe_idx < stripe_count) { u64 rand, cur_weight; cur_weight = 0; @@ -1682,35 +1750,36 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, rand = lu_prandom_u64_max(total_weight); - /* On average, this will hit larger-weighted tgts more often. - * 0-weight tgts will always get used last (only when rand=0) */ + /* On average, this will hit larger-weighted MDTs more often. + * 0-weight MDT will always get used last (only when rand=0) */ for (i = 0; i < pool->op_count; i++) { - __u32 idx = pool->op_array[i]; int rc2; - mdt = LTD_TGT(ltd, idx); + mdt_idx = pool->op_array[i]; + mdt = LTD_TGT(ltd, mdt_idx); if (!mdt->ltd_qos.ltq_usable) continue; cur_weight += mdt->ltd_qos.ltq_weight; - QOS_DEBUG("idx=%d nfound=%d cur_weight=%llu rand=%llu total_weight=%llu\n", - idx, nfound, cur_weight, rand, + QOS_DEBUG("stripe_count=%d stripe_index=%d cur_weight=%llu rand=%llu total_weight=%llu\n", + stripe_count, stripe_idx, cur_weight, rand, total_weight); if (cur_weight < rand) continue; - QOS_DEBUG("stripe=%d to idx=%d\n", nfound, idx); + QOS_DEBUG("stripe=%d to idx=%d\n", + stripe_idx, mdt_idx); - if (lod_qos_is_tgt_used(env, idx, nfound)) + if (lod_qos_is_tgt_used(env, mdt_idx, stripe_idx)) continue; - rc2 = obd_fid_alloc(env, mdt->ltd_exp, &fid, NULL); - if (rc2) { + rc2 = dt_fid_alloc(env, mdt->ltd_tgt, &fid, NULL, NULL); + if (rc2 < 0) { QOS_DEBUG("can't alloc FID on #%u: %d\n", - idx, rc2); + mdt_idx, rc2); continue; } @@ -1720,14 +1789,14 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, &conf); if (IS_ERR(dto)) { QOS_DEBUG("can't alloc stripe on #%u: %d\n", - idx, (int) PTR_ERR(dto)); + mdt_idx, (int) PTR_ERR(dto)); continue; } - lod_qos_tgt_in_use(env, nfound, idx); - stripes[nfound] = dto; + lod_qos_tgt_in_use(env, stripe_idx, mdt_idx); + stripes[stripe_idx] = dto; ltd_qos_update(ltd, mdt, &total_weight); - nfound++; + stripe_idx++; rc = 0; break; } @@ -1737,7 +1806,7 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, break; } - if (unlikely(nfound != stripe_count)) { + if (unlikely(stripe_idx != stripe_count)) { /* * when the decision to use weighted algorithm was made * we had enough appropriate OSPs, but this state can @@ -1746,20 +1815,20 @@ int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo, * an object due to just changed state */ QOS_DEBUG("%s: wanted %d objects, found only %d\n", - lod2obd(lod)->obd_name, stripe_count, nfound); - for (i = 1; i < nfound; i++) { + lod2obd(lod)->obd_name, stripe_count, stripe_idx); + for (i = saved_idx; i < stripe_idx; i++) { LASSERT(stripes[i] != NULL); dt_object_put(env, stripes[i]); stripes[i] = NULL; } /* makes sense to rebalance next time */ - ltd->ltd_qos.lq_dirty = 1; - ltd->ltd_qos.lq_same_space = 0; + set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags); + clear_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags); rc = -EAGAIN; } else { - rc = nfound; + rc = stripe_idx; } unlock: @@ -1781,19 +1850,20 @@ unlock: * * \param[in] lod LOD device * \param[in] lo The lod_object + * \param[in] comp_idx The component id, which the amount of stripes is + calculated for * \param[in] stripe_count count the caller would like to use * * \retval the maximum usable stripe count */ __u16 lod_get_stripe_count(struct lod_device *lod, struct lod_object *lo, - __u16 stripe_count, bool overstriping) + int comp_idx, __u16 stripe_count, bool overstriping) { __u32 max_stripes = LOV_MAX_STRIPE_COUNT_OLD; /* max stripe count is based on OSD ea size */ unsigned int easize = lod->lod_osd_max_easize; int i; - if (!stripe_count) stripe_count = lod->lod_ost_descs.ltd_lov_desc.ld_default_stripe_count; @@ -1817,9 +1887,17 @@ __u16 lod_get_stripe_count(struct lod_device *lod, struct lod_object *lo, lo->ldo_comp_cnt; for (i = 0; i < lo->ldo_comp_cnt; i++) { + unsigned int stripes; + + if (i == comp_idx) + continue; + lod_comp = &lo->ldo_comp_entries[i]; - comp_sz = lov_mds_md_size(lod_comp->llc_stripe_count, - LOV_MAGIC_V3); + /* Extension comp is never inited - 0 stripes on disk */ + stripes = lod_comp->llc_flags & LCME_FL_EXTENSION ? 0 : + lod_comp->llc_stripe_count; + + comp_sz = lov_mds_md_size(stripes, LOV_MAGIC_V3); total_comp_sz += comp_sz; if (lod_comp->llc_flags & LCME_FL_INIT) init_comp_sz += comp_sz; @@ -2047,7 +2125,7 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, else lod_free_comp_entries(lo); - rc = lod_verify_striping(d, lo, buf, false); + rc = lod_verify_striping(env, d, lo, buf, false); if (rc) RETURN(-EINVAL); @@ -2180,9 +2258,8 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, } lod_comp->llc_pattern = v1->lmm_pattern; - lod_comp->llc_stripe_size = desc->ld_default_stripe_size; - if (v1->lmm_stripe_size) - lod_comp->llc_stripe_size = v1->lmm_stripe_size; + lod_comp->llc_stripe_size = v1->lmm_stripe_size; + lod_adjust_stripe_size(lod_comp, desc->ld_default_stripe_size); lod_comp->llc_stripe_count = desc->ld_default_stripe_count; if (v1->lmm_stripe_count || @@ -2243,7 +2320,7 @@ int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo) { struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lod_avoid_guide *lag = &lod_env_info(env)->lti_avoid; - struct cfs_bitmap *bitmap = NULL; + unsigned long *bitmap = NULL; __u32 *new_oss = NULL; lag->lag_ost_avail = lod->lod_ost_count; @@ -2252,24 +2329,24 @@ int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo) lag->lag_oaa_count = 0; if (lag->lag_oss_avoid_array && lag->lag_oaa_size < lod->lod_ost_count) { - OBD_FREE(lag->lag_oss_avoid_array, - sizeof(__u32) * lag->lag_oaa_size); + OBD_FREE_PTR_ARRAY(lag->lag_oss_avoid_array, lag->lag_oaa_size); lag->lag_oss_avoid_array = NULL; lag->lag_oaa_size = 0; } /* init OST avoid guide bitmap */ if (lag->lag_ost_avoid_bitmap) { - if (lod->lod_ost_count <= lag->lag_ost_avoid_bitmap->size) { - CFS_RESET_BITMAP(lag->lag_ost_avoid_bitmap); + if (lod->lod_ost_count <= lag->lag_ost_avoid_size) { + bitmap_zero(lag->lag_ost_avoid_bitmap, + lag->lag_ost_avoid_size); } else { - CFS_FREE_BITMAP(lag->lag_ost_avoid_bitmap); + bitmap_free(lag->lag_ost_avoid_bitmap); lag->lag_ost_avoid_bitmap = NULL; } } if (!lag->lag_ost_avoid_bitmap) { - bitmap = CFS_ALLOCATE_BITMAP(lod->lod_ost_count); + bitmap = bitmap_zalloc(lod->lod_ost_count, GFP_KERNEL); if (!bitmap) return -ENOMEM; } @@ -2281,9 +2358,9 @@ int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo) * using OST count to allocate the array to store the OSS * id. */ - OBD_ALLOC(new_oss, sizeof(*new_oss) * lod->lod_ost_count); + OBD_ALLOC_PTR_ARRAY(new_oss, lod->lod_ost_count); if (!new_oss) { - CFS_FREE_BITMAP(bitmap); + bitmap_free(bitmap); return -ENOMEM; } } @@ -2292,8 +2369,10 @@ int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo) lag->lag_oss_avoid_array = new_oss; lag->lag_oaa_size = lod->lod_ost_count; } - if (bitmap) + if (bitmap) { lag->lag_ost_avoid_bitmap = bitmap; + lag->lag_ost_avoid_size = lod->lod_ost_count; + } return 0; } @@ -2307,7 +2386,7 @@ void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag, { struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lod_layout_component *lod_comp = &lo->ldo_comp_entries[comp_idx]; - struct cfs_bitmap *bitmap = lag->lag_ost_avoid_bitmap; + unsigned long *bitmap = lag->lag_ost_avoid_bitmap; int i, j; /* iterate mirrors */ @@ -2354,12 +2433,12 @@ void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag, ost = OST_TGT(lod, comp->llc_ost_indices[j]); lsq = ost->ltd_qos.ltq_svr; - if (cfs_bitmap_check(bitmap, ost->ltd_index)) + if (test_bit(ost->ltd_index, bitmap)) continue; QOS_DEBUG("OST%d used in conflicting mirror " "component\n", ost->ltd_index); - cfs_bitmap_set(bitmap, ost->ltd_index); + set_bit(ost->ltd_index, bitmap); lag->lag_ost_avail--; for (k = 0; k < lag->lag_oaa_count; k++) { @@ -2397,7 +2476,7 @@ void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag, */ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, struct lu_attr *attr, struct thandle *th, - int comp_idx) + int comp_idx, __u64 reserve) { struct lod_layout_component *lod_comp; struct lod_device *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev); @@ -2432,7 +2511,7 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, * could be changed if some OSTs are [de]activated manually. */ lod_qos_statfs_update(env, d, &d->lod_ost_descs); - stripe_len = lod_get_stripe_count(d, lo, + stripe_len = lod_get_stripe_count(d, lo, comp_idx, lod_comp->llc_stripe_count, lod_comp->llc_pattern & LOV_PATTERN_OVERSTRIPING); @@ -2440,13 +2519,14 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, if (stripe_len == 0) GOTO(out, rc = -ERANGE); lod_comp->llc_stripe_count = stripe_len; - OBD_ALLOC(stripe, sizeof(stripe[0]) * stripe_len); + OBD_ALLOC_PTR_ARRAY(stripe, stripe_len); if (stripe == NULL) GOTO(out, rc = -ENOMEM); - OBD_ALLOC(ost_indices, sizeof(*ost_indices) * stripe_len); + OBD_ALLOC_PTR_ARRAY(ost_indices, stripe_len); if (!ost_indices) GOTO(out, rc = -ENOMEM); +repeat: lod_getref(&d->lod_ost_descs); /* XXX: support for non-0 files w/o objects */ CDEBUG(D_OTHER, "tgt_count %d stripe_count %d\n", @@ -2455,7 +2535,7 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, if (lod_comp->llc_ostlist.op_array && lod_comp->llc_ostlist.op_count) { rc = lod_alloc_ost_list(env, lo, stripe, ost_indices, - th, comp_idx); + th, comp_idx, reserve); } else if (lod_comp->llc_stripe_offset == LOV_OFFSET_DEFAULT) { /** * collect OSTs and OSSs used in other mirrors whose @@ -2470,15 +2550,15 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, lod_collect_avoidance(lo, lag, comp_idx); rc = lod_ost_alloc_qos(env, lo, stripe, ost_indices, - flag, th, comp_idx); + flag, th, comp_idx, reserve); if (rc == -EAGAIN) rc = lod_ost_alloc_rr(env, lo, stripe, ost_indices, flag, th, - comp_idx); + comp_idx, reserve); } else { rc = lod_ost_alloc_specific(env, lo, stripe, ost_indices, flag, th, - comp_idx); + comp_idx, reserve); } put_ldts: lod_putref(d, &d->lod_ost_descs); @@ -2486,6 +2566,15 @@ put_ldts: for (i = 0; i < stripe_len; i++) if (stripe[i] != NULL) dt_object_put(env, stripe[i]); + + /* In case there is no space on any OST, let's ignore + * the @reserve space to avoid an error at the init + * time, probably the actual IO will be less than the + * given @reserve space (aka extension_size). */ + if (reserve) { + reserve = 0; + goto repeat; + } lod_comp->llc_stripe_count = 0; } else { lod_comp->llc_stripe = stripe; @@ -2522,10 +2611,9 @@ put_ldts: out: if (rc < 0) { if (stripe) - OBD_FREE(stripe, sizeof(stripe[0]) * stripe_len); + OBD_FREE_PTR_ARRAY(stripe, stripe_len); if (ost_indices) - OBD_FREE(ost_indices, - sizeof(*ost_indices) * stripe_len); + OBD_FREE_PTR_ARRAY(ost_indices, stripe_len); } RETURN(rc); } @@ -2577,7 +2665,7 @@ int lod_prepare_create(const struct lu_env *env, struct lod_object *lo, extent = &lod_comp->llc_extent; QOS_DEBUG("comp[%d] %lld "DEXT"\n", i, size, PEXT(extent)); if (!lo->ldo_is_composite || size >= extent->e_start) { - rc = lod_qos_prep_create(env, lo, attr, th, i); + rc = lod_qos_prep_create(env, lo, attr, th, i, 0); if (rc) break; }