Whamcloud - gitweb
git://git.whamcloud.com
/
fs
/
lustre-release.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
| inline |
side by side
LU-9859 lod: use linux kernel bitmap API
[fs/lustre-release.git]
/
lustre
/
lod
/
lod_qos.c
diff --git
a/lustre/lod/lod_qos.c
b/lustre/lod/lod_qos.c
index
a027947
..
f4a7783
100644
(file)
--- a/
lustre/lod/lod_qos.c
+++ b/
lustre/lod/lod_qos.c
@@
-65,17
+65,17
@@
static inline int lod_statfs_check(struct lu_tgt_descs *ltd,
{
struct obd_statfs *sfs = &tgt->ltd_statfs;
- if (((sfs->os_state & OS_STAT
E
_ENOSPC) ||
- (!ltd->ltd_is_mdt && sfs->os_state & OS_STAT
E
_ENOINO &&
+ if (((sfs->os_state & OS_STAT
FS
_ENOSPC) ||
+ (!ltd->ltd_is_mdt && sfs->os_state & OS_STAT
FS
_ENOINO &&
sfs->os_fprecreated == 0)))
return -ENOSPC;
/* If the OST is readonly then we can't allocate objects there */
- if (sfs->os_state & OS_STAT
E
_READONLY)
+ if (sfs->os_state & OS_STAT
FS
_READONLY)
return -EROFS;
/* object precreation is skipped on the OST with max_create_count=0 */
- if (!ltd->ltd_is_mdt && sfs->os_state & OS_STAT
E
_NOPRECREATE)
+ if (!ltd->ltd_is_mdt && sfs->os_state & OS_STAT
FS
_NOPRECREATE)
return -ENOBUFS;
return 0;
@@
-100,15
+100,18
@@
static inline int lod_statfs_check(struct lu_tgt_descs *ltd,
*/
static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
struct lu_tgt_descs *ltd,
- struct lu_tgt_desc *tgt)
+ struct lu_tgt_desc *tgt
, __u64 reserve
)
{
+ struct obd_statfs_info info = { 0 };
struct lov_desc *desc = <d->ltd_lov_desc;
int rc;
+ ENTRY;
LASSERT(d);
LASSERT(tgt);
- rc = dt_statfs(env, tgt->ltd_tgt, &tgt->ltd_statfs);
+ info.os_enable_pre = 1;
+ rc = dt_statfs_info(env, tgt->ltd_tgt, &tgt->ltd_statfs, &info);
if (rc && rc != -ENOTCONN)
CERROR("%s: statfs: rc = %d\n", lod2obd(d)->obd_name, rc);
@@
-118,6
+121,11
@@
static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
return rc;
}
+ if (reserve &&
+ (reserve + (info.os_reserved_mb_low << 20) >
+ tgt->ltd_statfs.os_bavail * tgt->ltd_statfs.os_bsize))
+ return -ENOSPC;
+
/* check whether device has changed state (active, inactive) */
if (rc != 0 && tgt->ltd_active) {
/* turned inactive? */
@@
-129,8
+137,8
@@
static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
LASSERT(desc->ld_active_tgt_count > 0);
desc->ld_active_tgt_count--;
-
ltd->ltd_qos.lq_dirty = 1
;
-
ltd->ltd_qos.lq_rr.lqr_dirty = 1
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags)
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_rr.lqr_flags)
;
CDEBUG(D_CONFIG, "%s: turns inactive\n",
tgt->ltd_exp->exp_obd->obd_name);
}
@@
-145,15
+153,22
@@
static int lod_statfs_and_check(const struct lu_env *env, struct lod_device *d,
tgt->ltd_active = 1;
tgt->ltd_connecting = 0;
desc->ld_active_tgt_count++;
-
ltd->ltd_qos.lq_dirty = 1
;
-
ltd->ltd_qos.lq_rr.lqr_dirty = 1
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags)
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_rr.lqr_flags)
;
CDEBUG(D_CONFIG, "%s: turns active\n",
tgt->ltd_exp->exp_obd->obd_name);
}
spin_unlock(&d->lod_lock);
}
+ if (rc == -ENOTCONN) {
+ /* In case that the ENOTCONN for inactive OST state is
+ * mistreated as MDT disconnection state by the client,
+ * this error should be changed to someone else.
+ */
+ rc = -EREMOTEIO;
+ }
-
return rc
;
+
RETURN(rc)
;
}
static int lod_is_tgt_usable(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt)
@@
-203,12
+218,12
@@
void lod_qos_statfs_update(const struct lu_env *env, struct lod_device *lod,
ltd_foreach_tgt(ltd, tgt) {
avail = tgt->ltd_statfs.os_bavail;
- if (lod_statfs_and_check(env, lod, ltd, tgt))
+ if (lod_statfs_and_check(env, lod, ltd, tgt
, 0
))
continue;
if (tgt->ltd_statfs.os_bavail != avail)
/* recalculate weigths */
-
ltd->ltd_qos.lq_dirty = 1
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags)
;
}
obd->obd_osfs_age = ktime_get_seconds();
@@
-247,7
+262,7
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
int rc;
ENTRY;
- if (!
lqr->lqr_dirty
) {
+ if (!
test_bit(LQ_DIRTY, &lqr->lqr_flags)
) {
LASSERT(lqr->lqr_pool.op_size);
RETURN(0);
}
@@
-259,7
+274,7
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
* Check again. While we were sleeping on @lq_rw_sem something could
* change.
*/
- if (!
lqr->lqr_dirty
) {
+ if (!
test_bit(LQ_DIRTY, &lqr->lqr_flags)
) {
LASSERT(lqr->lqr_pool.op_size);
up_write(<d->ltd_qos.lq_rw_sem);
RETURN(0);
@@
-272,7
+287,7
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
deleting from the pool. The lq_rw_sem insures that nobody else
is reading. */
lqr->lqr_pool.op_count = real_count;
- rc =
lod_
tgt_pool_extend(&lqr->lqr_pool, real_count);
+ rc = tgt_pool_extend(&lqr->lqr_pool, real_count);
if (rc) {
up_write(<d->ltd_qos.lq_rw_sem);
RETURN(rc);
@@
-288,8
+303,8
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
for (i = 0; i < lqr->lqr_pool.op_count; i++) {
int next;
- if (!
cfs_bitmap_check(ltd->ltd_tgt_bitmap
,
-
src_pool->op_array[i]
))
+ if (!
test_bit(src_pool->op_array[i]
,
+
ltd->ltd_tgt_bitmap
))
continue;
tgt = LTD_TGT(ltd, src_pool->op_array[i]);
@@
-308,7
+323,7
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
}
}
-
lqr->lqr_dirty = 0
;
+
clear_bit(LQ_DIRTY, &lqr->lqr_flags)
;
up_write(<d->ltd_qos.lq_rw_sem);
if (placed != real_count) {
@@
-320,7
+335,7
@@
static int lod_qos_calc_rr(struct lod_device *lod, struct lu_tgt_descs *ltd,
LCONSOLE(D_WARNING, "rr #%d tgt idx=%d\n", i,
lqr->lqr_pool.op_array[i]);
}
-
lqr->lqr_dirty = 1
;
+
set_bit(LQ_DIRTY, &lqr->lqr_flags)
;
RETURN(-EAGAIN);
}
@@
-379,7
+394,7
@@
static struct dt_object *lod_qos_declare_object_on(const struct lu_env *env,
*/
o = lu_object_anon(env, nd, NULL);
if (IS_ERR(o))
- GOTO(out, dt = ERR_
PTR(PTR_ERR(o)
));
+ GOTO(out, dt = ERR_
CAST(o
));
n = lu_object_locate(o->lo_header, nd->ld_type);
if (unlikely(n == NULL)) {
@@
-442,7
+457,7
@@
static inline int lod_qos_tgt_in_use_clear(const struct lu_env *env,
if (info->lti_ea_store_size < sizeof(int) * stripes)
lod_ea_store_resize(info, stripes * sizeof(int));
if (info->lti_ea_store_size < sizeof(int) * stripes) {
- CERROR("can't allocate memory for
os
t-in-use array\n");
+ CERROR("can't allocate memory for
tg
t-in-use array\n");
return -ENOMEM;
}
memset(info->lti_ea_store, -1, sizeof(int) * stripes);
@@
-564,7
+579,7
@@
static inline bool lod_should_avoid_ost(struct lod_object *lo,
bool used = false;
int i;
- if (!
cfs_bitmap_check(lod->lod_ost_bitmap, index
)) {
+ if (!
test_bit(index, lod->lod_ost_bitmap
)) {
QOS_DEBUG("OST%d: been used in conflicting mirror component\n",
index);
return true;
@@
-592,7
+607,7
@@
static inline bool lod_should_avoid_ost(struct lod_object *lo,
return false;
/* if the OSS has been used, check whether the OST has been used */
- if (!
cfs_bitmap_check(lag->lag_ost_avoid_bitmap, index
))
+ if (!
test_bit(index, lag->lag_ost_avoid_bitmap
))
used = false;
else
QOS_DEBUG("OST%d: been used in conflicting mirror component\n",
@@
-607,7
+622,8
@@
static int lod_check_and_reserve_ost(const struct lu_env *env,
struct dt_object **stripe,
__u32 *ost_indices,
struct thandle *th,
- bool *overstriped)
+ bool *overstriped,
+ __u64 reserve)
{
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
struct lod_avoid_guide *lag = &lod_env_info(env)->lti_avoid;
@@
-618,7
+634,7
@@
static int lod_check_and_reserve_ost(const struct lu_env *env,
ENTRY;
- rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost);
+ rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost
, reserve
);
if (rc)
RETURN(rc);
@@
-634,7
+650,7
@@
static int lod_check_and_reserve_ost(const struct lu_env *env,
/*
* try to use another OSP if this one is degraded
*/
- if (ost->ltd_statfs.os_state & OS_STAT
E
_DEGRADED && speed < 2) {
+ if (ost->ltd_statfs.os_state & OS_STAT
FS
_DEGRADED && speed < 2) {
QOS_DEBUG("#%d: degraded\n", ost_idx);
RETURN(rc);
}
@@
-717,7
+733,8
@@
static int lod_check_and_reserve_ost(const struct lu_env *env,
*/
static int lod_ost_alloc_rr(const struct lu_env *env, struct lod_object *lo,
struct dt_object **stripe, __u32 *ost_indices,
- int flags, struct thandle *th, int comp_idx)
+ int flags, struct thandle *th, int comp_idx,
+ __u64 reserve)
{
struct lod_layout_component *lod_comp;
struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
@@
-800,7
+817,7
@@
repeat_find:
stripe_idx, array_idx, ost_idx);
if ((ost_idx == LOV_QOS_EMPTY) ||
- !
cfs_bitmap_check(m->lod_ost_bitmap, ost_idx
))
+ !
test_bit(ost_idx, m->lod_ost_bitmap
))
continue;
/* Fail Check before osc_precreate() is called
@@
-811,7
+828,8
@@
repeat_find:
spin_unlock(&lqr->lqr_alloc);
rc = lod_check_and_reserve_ost(env, lo, lod_comp, ost_idx,
speed, &stripe_idx, stripe,
- ost_indices, th, &overstriped);
+ ost_indices, th, &overstriped,
+ reserve);
spin_lock(&lqr->lqr_alloc);
if (rc != 0 && OST_TGT(m, ost_idx)->ltd_connecting)
@@
-857,6
+875,44
@@
out:
RETURN(rc);
}
+static int
+lod_qos_mdt_in_use_init(const struct lu_env *env,
+ const struct lu_tgt_descs *ltd,
+ u32 stripe_idx, u32 stripe_count,
+ const struct lu_tgt_pool *pool,
+ struct dt_object **stripes)
+{
+ u32 mdt_idx;
+ struct lu_tgt_desc *mdt;
+ int i, j;
+ int rc;
+
+ rc = lod_qos_tgt_in_use_clear(env, stripe_count);
+ if (rc)
+ return rc;
+
+ /* if stripe_idx > 1, we are splitting directory, mark existing stripes
+ * in_use. Because for either split or creation, stripe 0 is local,
+ * don't mark it in use.
+ */
+ for (i = 1; i < stripe_idx; i++) {
+ LASSERT(stripes[i]);
+ for (j = 0; j < pool->op_count; j++) {
+ mdt_idx = pool->op_array[j];
+
+ if (!test_bit(mdt_idx, ltd->ltd_tgt_bitmap))
+ continue;
+
+ mdt = LTD_TGT(ltd, mdt_idx);
+ if (&mdt->ltd_tgt->dd_lu_dev ==
+ stripes[i]->do_lu.lo_dev)
+ lod_qos_tgt_in_use(env, i, mdt_idx);
+ }
+ }
+
+ return 0;
+}
+
/**
* Allocate a striping using round-robin algorithm.
*
@@
-870,7
+926,7
@@
out:
*
* \param[in] env execution environment for this thread
* \param[in] lo LOD object
- * \param[out] stripe
striping created
+ * \param[out] stripe
s
striping created
*
* \retval positive stripe objects allocated, including the first stripe
* allocated outside
@@
-878,7
+934,8
@@
out:
* \retval negative negated errno for other failures
*/
int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo,
- struct dt_object **stripe)
+ struct dt_object **stripes, u32 stripe_idx,
+ u32 stripe_count)
{
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
struct lu_tgt_descs *ltd = &lod->lod_mdt_descs;
@@
-890,9
+947,8
@@
int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo,
struct dt_object *dto;
unsigned int pool_idx;
unsigned int i;
- u32 start_idx_temp;
- u32 stripe_count = lo->ldo_dir_stripe_count;
- u32 stripe_idx = 1;
+ u32 saved_idx = stripe_idx;
+ u32 start_mdt;
u32 mdt_idx;
bool use_degraded = false;
int tgt_connecting = 0;
@@
-906,7
+962,8
@@
int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo,
if (rc)
RETURN(rc);
- rc = lod_qos_tgt_in_use_clear(env, stripe_count);
+ rc = lod_qos_mdt_in_use_init(env, ltd, stripe_idx, stripe_count, pool,
+ stripes);
if (rc)
RETURN(rc);
@@
-927,10
+984,10
@@
int lod_mdt_alloc_rr(const struct lu_env *env, struct lod_object *lo,
(pool->op_count % (stripe_count - 1)) != 1)
++lqr->lqr_offset_idx;
}
- start_
idx_temp
= lqr->lqr_start_idx;
+ start_
mdt
= lqr->lqr_start_idx;
repeat_find:
- QOS_DEBUG("want
%d start_idx %d start_count %d offset %d active %d count
%d\n",
+ QOS_DEBUG("want
=%d start_idx=%d start_count=%d offset=%d active=%d count=
%d\n",
stripe_count - 1, lqr->lqr_start_idx, lqr->lqr_start_count,
lqr->lqr_offset_idx, pool->op_count, pool->op_count);
@@
-946,7
+1003,7
@@
repeat_find:
stripe_idx, pool_idx, mdt_idx);
if (mdt_idx == LOV_QOS_EMPTY ||
- !
cfs_bitmap_check(ltd->ltd_tgt_bitmap, mdt_idx
))
+ !
test_bit(mdt_idx, ltd->ltd_tgt_bitmap
))
continue;
/* do not put >1 objects on one MDT */
@@
-961,15
+1018,15
@@
repeat_find:
}
/* try to use another OSP if this one is degraded */
- if (mdt->ltd_statfs.os_state & OS_STAT
E
_DEGRADED &&
+ if (mdt->ltd_statfs.os_state & OS_STAT
FS
_DEGRADED &&
!use_degraded) {
QOS_DEBUG("#%d: degraded\n", mdt_idx);
continue;
}
spin_unlock(&lqr->lqr_alloc);
- rc =
obd_fid_alloc(env, mdt->ltd_exp, &fid
, NULL);
- if (rc) {
+ rc =
dt_fid_alloc(env, mdt->ltd_tgt, &fid, NULL
, NULL);
+ if (rc
< 0
) {
QOS_DEBUG("#%d: alloc FID failed: %dl\n", mdt_idx, rc);
spin_lock(&lqr->lqr_alloc);
continue;
@@
-990,14
+1047,13
@@
repeat_find:
}
lod_qos_tgt_in_use(env, stripe_idx, mdt_idx);
- stripe[stripe_idx] = dto;
- stripe_idx++;
+ stripes[stripe_idx++] = dto;
}
if (!use_degraded && stripe_idx < stripe_count) {
- /* Try again, allowing slower
OSC
s */
+ /* Try again, allowing slower
MDT
s */
use_degraded = true;
- lqr->lqr_start_idx = start_
idx_temp
;
+ lqr->lqr_start_idx = start_
mdt
;
tgt_connecting = 0;
goto repeat_find;
@@
-1005,7
+1061,7
@@
repeat_find:
spin_unlock(&lqr->lqr_alloc);
up_read(<d->ltd_qos.lq_rw_sem);
- if (stripe_idx >
1
)
+ if (stripe_idx >
saved_idx
)
/* at least one stripe is allocated */
RETURN(stripe_idx);
@@
-1043,7
+1099,7
@@
repeat_find:
*/
static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo,
struct dt_object **stripe, __u32 *ost_indices,
- struct thandle *th, int comp_idx)
+ struct thandle *th, int comp_idx
, __u64 reserve
)
{
struct lod_layout_component *lod_comp;
struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
@@
-1086,7
+1142,7
@@
static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo,
i++, array_idx = (array_idx + 1) % lod_comp->llc_stripe_count) {
__u32 ost_idx = lod_comp->llc_ostlist.op_array[array_idx];
- if (!
cfs_bitmap_check(m->lod_ost_bitmap, ost_idx
)) {
+ if (!
test_bit(ost_idx, m->lod_ost_bitmap
)) {
rc = -ENODEV;
break;
}
@@
-1101,7
+1157,8
@@
static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo,
}
rc = lod_statfs_and_check(env, m, &m->lod_ost_descs,
- LTD_TGT(&m->lod_ost_descs, ost_idx));
+ LTD_TGT(&m->lod_ost_descs, ost_idx),
+ reserve);
if (rc < 0) /* this OSP doesn't feel well */
break;
@@
-1155,7
+1212,8
@@
static int lod_alloc_ost_list(const struct lu_env *env, struct lod_object *lo,
static int lod_ost_alloc_specific(const struct lu_env *env,
struct lod_object *lo,
struct dt_object **stripe, __u32 *ost_indices,
- int flags, struct thandle *th, int comp_idx)
+ int flags, struct thandle *th, int comp_idx,
+ __u64 reserve)
{
struct lod_layout_component *lod_comp;
struct lod_device *m = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
@@
-1214,7
+1272,7
@@
repeat_find:
i++, array_idx = (array_idx + 1) % ost_count) {
ost_idx = osts->op_array[array_idx];
- if (!
cfs_bitmap_check(m->lod_ost_bitmap, ost_idx
))
+ if (!
test_bit(ost_idx, m->lod_ost_bitmap
))
continue;
/* Fail Check before osc_precreate() is called
@@
-1248,7
+1306,8
@@
repeat_find:
* start OST, then it can be skipped, otherwise skip it only
* if it is inactive/recovering/out-of-space." */
- rc = lod_statfs_and_check(env, m, &m->lod_ost_descs, tgt);
+ rc = lod_statfs_and_check(env, m, &m->lod_ost_descs,
+ tgt, reserve);
if (rc) {
/* this OSP doesn't feel well */
continue;
@@
-1349,7
+1408,8
@@
out:
*/
static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo,
struct dt_object **stripe, __u32 *ost_indices,
- int flags, struct thandle *th, int comp_idx)
+ int flags, struct thandle *th, int comp_idx,
+ __u64 reserve)
{
struct lod_layout_component *lod_comp;
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
@@
-1412,19
+1472,20
@@
static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo,
good_osts = 0;
/* Find all the OSTs that are valid stripe candidates */
for (i = 0; i < osts->op_count; i++) {
- if (!
cfs_bitmap_check(lod->lod_ost_bitmap, osts->op_array[i]
))
+ if (!
test_bit(osts->op_array[i], lod->lod_ost_bitmap
))
continue;
ost = OST_TGT(lod, osts->op_array[i]);
ost->ltd_qos.ltq_usable = 0;
- rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs, ost);
+ rc = lod_statfs_and_check(env, lod, &lod->lod_ost_descs,
+ ost, reserve);
if (rc) {
/* this OSP doesn't feel well */
continue;
}
- if (ost->ltd_statfs.os_state & OS_STAT
E
_DEGRADED)
+ if (ost->ltd_statfs.os_state & OS_STAT
FS
_DEGRADED)
continue;
/* Fail Check before osc_precreate() is called
@@
-1542,9
+1603,8
@@
static int lod_ost_alloc_qos(const struct lu_env *env, struct lod_object *lo,
}
/* makes sense to rebalance next time */
- lod->lod_ost_descs.ltd_qos.lq_dirty = 1;
- lod->lod_ost_descs.ltd_qos.lq_same_space = 0;
-
+ set_bit(LQ_DIRTY, &lod->lod_ost_descs.ltd_qos.lq_flags);
+ clear_bit(LQ_SAME_SPACE, &lod->lod_ost_descs.ltd_qos.lq_flags);
rc = -EAGAIN;
}
@@
-1590,6
+1650,9
@@
out_nolock:
*
* \param[in] env execution environment for this thread
* \param[in] lo LOD object
+ * \param[in] stripe_idx starting stripe index to allocate, if it's not
+ * 0, we are restriping directory
+ * \param[in] stripe_count total stripe count
* \param[out] stripes striping created
*
* \retval positive stripes allocated, and it should be equal to
@@
-1599,7
+1662,8
@@
out_nolock:
* \retval negative errno on failure
*/
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
- struct dt_object **stripes)
+ struct dt_object **stripes, u32 stripe_idx,
+ u32 stripe_count)
{
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
struct lu_tgt_descs *ltd = &lod->lod_mdt_descs;
@@
-1609,23
+1673,32
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
struct lu_tgt_desc *mdt;
struct dt_object *dto;
u64 total_weight = 0;
- u32 s
tripe_count = lo->ldo_dir_stripe_count
;
- u
nsigned int nfound
;
+ u32 s
aved_idx = stripe_idx
;
+ u
32 mdt_idx
;
unsigned int good_mdts;
unsigned int i;
int rc = 0;
ENTRY;
- if (stripe_count == 1)
- RETURN(1);
+ LASSERT(stripe_idx <= stripe_count);
+ if (stripe_idx == stripe_count)
+ RETURN(stripe_count);
+ /* use MDT pool in @ltd, once MDT pool is supported in the future, it
+ * can be passed in as argument like OST object allocation.
+ */
pool = <d->ltd_tgt_pool;
/* Detect -EAGAIN early, before expensive lock is taken. */
if (!ltd_qos_is_usable(ltd))
RETURN(-EAGAIN);
+ rc = lod_qos_mdt_in_use_init(env, ltd, stripe_idx, stripe_count, pool,
+ stripes);
+ if (rc)
+ RETURN(rc);
+
/* Do actual allocation, use write lock here. */
down_write(<d->ltd_qos.lq_rw_sem);
@@
-1640,14
+1713,10
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
if (rc)
GOTO(unlock, rc);
- rc = lod_qos_tgt_in_use_clear(env, stripe_count);
- if (rc)
- GOTO(unlock, rc);
-
good_mdts = 0;
- /* Find all the
tgt
s that are valid stripe candidates */
+ /* Find all the
MDT
s that are valid stripe candidates */
for (i = 0; i < pool->op_count; i++) {
- if (!
cfs_bitmap_check(ltd->ltd_tgt_bitmap, pool->op_array[i]
))
+ if (!
test_bit(pool->op_array[i], ltd->ltd_tgt_bitmap
))
continue;
mdt = LTD_TGT(ltd, pool->op_array[i]);
@@
-1657,7
+1726,7
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
if (rc)
continue;
- if (mdt->ltd_statfs.os_state & OS_STAT
E
_DEGRADED)
+ if (mdt->ltd_statfs.os_state & OS_STAT
FS
_DEGRADED)
continue;
mdt->ltd_qos.ltq_usable = 1;
@@
-1667,14
+1736,13
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
good_mdts++;
}
- QOS_DEBUG("found %d good
tgt
s\n", good_mdts);
+ QOS_DEBUG("found %d good
MDT
s\n", good_mdts);
- if (good_mdts < stripe_count -
1
)
+ if (good_mdts < stripe_count -
stripe_idx
)
GOTO(unlock, rc = -EAGAIN);
- /* Find enough tgts with weighted random allocation. */
- nfound = 1;
- while (nfound < stripe_count) {
+ /* Find enough MDTs with weighted random allocation. */
+ while (stripe_idx < stripe_count) {
u64 rand, cur_weight;
cur_weight = 0;
@@
-1682,35
+1750,36
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
rand = lu_prandom_u64_max(total_weight);
- /* On average, this will hit larger-weighted
tgt
s more often.
- * 0-weight
tgts
will always get used last (only when rand=0) */
+ /* On average, this will hit larger-weighted
MDT
s more often.
+ * 0-weight
MDT
will always get used last (only when rand=0) */
for (i = 0; i < pool->op_count; i++) {
- __u32 idx = pool->op_array[i];
int rc2;
- mdt = LTD_TGT(ltd, idx);
+ mdt_idx = pool->op_array[i];
+ mdt = LTD_TGT(ltd, mdt_idx);
if (!mdt->ltd_qos.ltq_usable)
continue;
cur_weight += mdt->ltd_qos.ltq_weight;
- QOS_DEBUG("
idx=%d nfound
=%d cur_weight=%llu rand=%llu total_weight=%llu\n",
-
idx, nfound
, cur_weight, rand,
+ QOS_DEBUG("
stripe_count=%d stripe_index
=%d cur_weight=%llu rand=%llu total_weight=%llu\n",
+
stripe_count, stripe_idx
, cur_weight, rand,
total_weight);
if (cur_weight < rand)
continue;
- QOS_DEBUG("stripe=%d to idx=%d\n", nfound, idx);
+ QOS_DEBUG("stripe=%d to idx=%d\n",
+ stripe_idx, mdt_idx);
- if (lod_qos_is_tgt_used(env,
idx, nfound
))
+ if (lod_qos_is_tgt_used(env,
mdt_idx, stripe_idx
))
continue;
- rc2 =
obd_fid_alloc(env, mdt->ltd_exp, &fid
, NULL);
- if (rc2) {
+ rc2 =
dt_fid_alloc(env, mdt->ltd_tgt, &fid, NULL
, NULL);
+ if (rc2
< 0
) {
QOS_DEBUG("can't alloc FID on #%u: %d\n",
- idx, rc2);
+
mdt_
idx, rc2);
continue;
}
@@
-1720,14
+1789,14
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
&conf);
if (IS_ERR(dto)) {
QOS_DEBUG("can't alloc stripe on #%u: %d\n",
- idx, (int) PTR_ERR(dto));
+
mdt_
idx, (int) PTR_ERR(dto));
continue;
}
- lod_qos_tgt_in_use(env,
nfound,
idx);
- stripes[
nfound
] = dto;
+ lod_qos_tgt_in_use(env,
stripe_idx, mdt_
idx);
+ stripes[
stripe_idx
] = dto;
ltd_qos_update(ltd, mdt, &total_weight);
-
nfound
++;
+
stripe_idx
++;
rc = 0;
break;
}
@@
-1737,7
+1806,7
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
break;
}
- if (unlikely(
nfound
!= stripe_count)) {
+ if (unlikely(
stripe_idx
!= stripe_count)) {
/*
* when the decision to use weighted algorithm was made
* we had enough appropriate OSPs, but this state can
@@
-1746,20
+1815,20
@@
int lod_mdt_alloc_qos(const struct lu_env *env, struct lod_object *lo,
* an object due to just changed state
*/
QOS_DEBUG("%s: wanted %d objects, found only %d\n",
- lod2obd(lod)->obd_name, stripe_count,
nfound
);
- for (i =
1; i < nfound
; i++) {
+ lod2obd(lod)->obd_name, stripe_count,
stripe_idx
);
+ for (i =
saved_idx; i < stripe_idx
; i++) {
LASSERT(stripes[i] != NULL);
dt_object_put(env, stripes[i]);
stripes[i] = NULL;
}
/* makes sense to rebalance next time */
-
ltd->ltd_qos.lq_dirty = 1
;
-
ltd->ltd_qos.lq_same_space = 0
;
+
set_bit(LQ_DIRTY, <d->ltd_qos.lq_flags)
;
+
clear_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags)
;
rc = -EAGAIN;
} else {
- rc =
nfound
;
+ rc =
stripe_idx
;
}
unlock:
@@
-1781,19
+1850,20
@@
unlock:
*
* \param[in] lod LOD device
* \param[in] lo The lod_object
+ * \param[in] comp_idx The component id, which the amount of stripes is
+ calculated for
* \param[in] stripe_count count the caller would like to use
*
* \retval the maximum usable stripe count
*/
__u16 lod_get_stripe_count(struct lod_device *lod, struct lod_object *lo,
- __u16 stripe_count, bool overstriping)
+
int comp_idx,
__u16 stripe_count, bool overstriping)
{
__u32 max_stripes = LOV_MAX_STRIPE_COUNT_OLD;
/* max stripe count is based on OSD ea size */
unsigned int easize = lod->lod_osd_max_easize;
int i;
-
if (!stripe_count)
stripe_count =
lod->lod_ost_descs.ltd_lov_desc.ld_default_stripe_count;
@@
-1817,9
+1887,17
@@
__u16 lod_get_stripe_count(struct lod_device *lod, struct lod_object *lo,
lo->ldo_comp_cnt;
for (i = 0; i < lo->ldo_comp_cnt; i++) {
+ unsigned int stripes;
+
+ if (i == comp_idx)
+ continue;
+
lod_comp = &lo->ldo_comp_entries[i];
- comp_sz = lov_mds_md_size(lod_comp->llc_stripe_count,
- LOV_MAGIC_V3);
+ /* Extension comp is never inited - 0 stripes on disk */
+ stripes = lod_comp->llc_flags & LCME_FL_EXTENSION ? 0 :
+ lod_comp->llc_stripe_count;
+
+ comp_sz = lov_mds_md_size(stripes, LOV_MAGIC_V3);
total_comp_sz += comp_sz;
if (lod_comp->llc_flags & LCME_FL_INIT)
init_comp_sz += comp_sz;
@@
-2047,7
+2125,7
@@
int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo,
else
lod_free_comp_entries(lo);
- rc = lod_verify_striping(d, lo, buf, false);
+ rc = lod_verify_striping(
env,
d, lo, buf, false);
if (rc)
RETURN(-EINVAL);
@@
-2180,9
+2258,8
@@
int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo,
}
lod_comp->llc_pattern = v1->lmm_pattern;
- lod_comp->llc_stripe_size = desc->ld_default_stripe_size;
- if (v1->lmm_stripe_size)
- lod_comp->llc_stripe_size = v1->lmm_stripe_size;
+ lod_comp->llc_stripe_size = v1->lmm_stripe_size;
+ lod_adjust_stripe_size(lod_comp, desc->ld_default_stripe_size);
lod_comp->llc_stripe_count = desc->ld_default_stripe_count;
if (v1->lmm_stripe_count ||
@@
-2243,7
+2320,7
@@
int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo)
{
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
struct lod_avoid_guide *lag = &lod_env_info(env)->lti_avoid;
-
struct cfs_bitmap
*bitmap = NULL;
+
unsigned long
*bitmap = NULL;
__u32 *new_oss = NULL;
lag->lag_ost_avail = lod->lod_ost_count;
@@
-2252,24
+2329,24
@@
int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo)
lag->lag_oaa_count = 0;
if (lag->lag_oss_avoid_array &&
lag->lag_oaa_size < lod->lod_ost_count) {
- OBD_FREE(lag->lag_oss_avoid_array,
- sizeof(__u32) * lag->lag_oaa_size);
+ OBD_FREE_PTR_ARRAY(lag->lag_oss_avoid_array, lag->lag_oaa_size);
lag->lag_oss_avoid_array = NULL;
lag->lag_oaa_size = 0;
}
/* init OST avoid guide bitmap */
if (lag->lag_ost_avoid_bitmap) {
- if (lod->lod_ost_count <= lag->lag_ost_avoid_bitmap->size) {
- CFS_RESET_BITMAP(lag->lag_ost_avoid_bitmap);
+ if (lod->lod_ost_count <= lag->lag_ost_avoid_size) {
+ bitmap_zero(lag->lag_ost_avoid_bitmap,
+ lag->lag_ost_avoid_size);
} else {
-
CFS_FREE_BITMAP
(lag->lag_ost_avoid_bitmap);
+
bitmap_free
(lag->lag_ost_avoid_bitmap);
lag->lag_ost_avoid_bitmap = NULL;
}
}
if (!lag->lag_ost_avoid_bitmap) {
- bitmap =
CFS_ALLOCATE_BITMAP(lod->lod_ost_count
);
+ bitmap =
bitmap_zalloc(lod->lod_ost_count, GFP_KERNEL
);
if (!bitmap)
return -ENOMEM;
}
@@
-2281,9
+2358,9
@@
int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo)
* using OST count to allocate the array to store the OSS
* id.
*/
- OBD_ALLOC
(new_oss, sizeof(*new_oss) *
lod->lod_ost_count);
+ OBD_ALLOC
_PTR_ARRAY(new_oss,
lod->lod_ost_count);
if (!new_oss) {
-
CFS_FREE_BITMAP
(bitmap);
+
bitmap_free
(bitmap);
return -ENOMEM;
}
}
@@
-2292,8
+2369,10
@@
int lod_prepare_avoidance(const struct lu_env *env, struct lod_object *lo)
lag->lag_oss_avoid_array = new_oss;
lag->lag_oaa_size = lod->lod_ost_count;
}
- if (bitmap)
+ if (bitmap)
{
lag->lag_ost_avoid_bitmap = bitmap;
+ lag->lag_ost_avoid_size = lod->lod_ost_count;
+ }
return 0;
}
@@
-2307,7
+2386,7
@@
void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag,
{
struct lod_device *lod = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
struct lod_layout_component *lod_comp = &lo->ldo_comp_entries[comp_idx];
-
struct cfs_bitmap
*bitmap = lag->lag_ost_avoid_bitmap;
+
unsigned long
*bitmap = lag->lag_ost_avoid_bitmap;
int i, j;
/* iterate mirrors */
@@
-2354,12
+2433,12
@@
void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag,
ost = OST_TGT(lod, comp->llc_ost_indices[j]);
lsq = ost->ltd_qos.ltq_svr;
- if (
cfs_bitmap_check(bitmap, ost->ltd_index
))
+ if (
test_bit(ost->ltd_index, bitmap
))
continue;
QOS_DEBUG("OST%d used in conflicting mirror "
"component\n", ost->ltd_index);
-
cfs_bitmap_set(bitmap, ost->ltd_index
);
+
set_bit(ost->ltd_index, bitmap
);
lag->lag_ost_avail--;
for (k = 0; k < lag->lag_oaa_count; k++) {
@@
-2397,7
+2476,7
@@
void lod_collect_avoidance(struct lod_object *lo, struct lod_avoid_guide *lag,
*/
int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
struct lu_attr *attr, struct thandle *th,
- int comp_idx)
+ int comp_idx
, __u64 reserve
)
{
struct lod_layout_component *lod_comp;
struct lod_device *d = lu2lod_dev(lod2lu_obj(lo)->lo_dev);
@@
-2432,7
+2511,7
@@
int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
* could be changed if some OSTs are [de]activated manually.
*/
lod_qos_statfs_update(env, d, &d->lod_ost_descs);
- stripe_len = lod_get_stripe_count(d, lo,
+ stripe_len = lod_get_stripe_count(d, lo,
comp_idx,
lod_comp->llc_stripe_count,
lod_comp->llc_pattern &
LOV_PATTERN_OVERSTRIPING);
@@
-2440,13
+2519,14
@@
int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
if (stripe_len == 0)
GOTO(out, rc = -ERANGE);
lod_comp->llc_stripe_count = stripe_len;
- OBD_ALLOC
(stripe, sizeof(stripe[0]) *
stripe_len);
+ OBD_ALLOC
_PTR_ARRAY(stripe,
stripe_len);
if (stripe == NULL)
GOTO(out, rc = -ENOMEM);
- OBD_ALLOC
(ost_indices, sizeof(*ost_indices) *
stripe_len);
+ OBD_ALLOC
_PTR_ARRAY(ost_indices,
stripe_len);
if (!ost_indices)
GOTO(out, rc = -ENOMEM);
+repeat:
lod_getref(&d->lod_ost_descs);
/* XXX: support for non-0 files w/o objects */
CDEBUG(D_OTHER, "tgt_count %d stripe_count %d\n",
@@
-2455,7
+2535,7
@@
int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
if (lod_comp->llc_ostlist.op_array &&
lod_comp->llc_ostlist.op_count) {
rc = lod_alloc_ost_list(env, lo, stripe, ost_indices,
- th, comp_idx);
+ th, comp_idx
, reserve
);
} else if (lod_comp->llc_stripe_offset == LOV_OFFSET_DEFAULT) {
/**
* collect OSTs and OSSs used in other mirrors whose
@@
-2470,15
+2550,15
@@
int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo,
lod_collect_avoidance(lo, lag, comp_idx);
rc = lod_ost_alloc_qos(env, lo, stripe, ost_indices,
- flag, th, comp_idx);
+ flag, th, comp_idx
, reserve
);
if (rc == -EAGAIN)
rc = lod_ost_alloc_rr(env, lo, stripe,
ost_indices, flag, th,
- comp_idx);
+ comp_idx
, reserve
);
} else {
rc = lod_ost_alloc_specific(env, lo, stripe,
ost_indices, flag, th,
- comp_idx);
+ comp_idx
, reserve
);
}
put_ldts:
lod_putref(d, &d->lod_ost_descs);
@@
-2486,6
+2566,15
@@
put_ldts:
for (i = 0; i < stripe_len; i++)
if (stripe[i] != NULL)
dt_object_put(env, stripe[i]);
+
+ /* In case there is no space on any OST, let's ignore
+ * the @reserve space to avoid an error at the init
+ * time, probably the actual IO will be less than the
+ * given @reserve space (aka extension_size). */
+ if (reserve) {
+ reserve = 0;
+ goto repeat;
+ }
lod_comp->llc_stripe_count = 0;
} else {
lod_comp->llc_stripe = stripe;
@@
-2522,10
+2611,9
@@
put_ldts:
out:
if (rc < 0) {
if (stripe)
- OBD_FREE
(stripe, sizeof(stripe[0]) *
stripe_len);
+ OBD_FREE
_PTR_ARRAY(stripe,
stripe_len);
if (ost_indices)
- OBD_FREE(ost_indices,
- sizeof(*ost_indices) * stripe_len);
+ OBD_FREE_PTR_ARRAY(ost_indices, stripe_len);
}
RETURN(rc);
}
@@
-2577,7
+2665,7
@@
int lod_prepare_create(const struct lu_env *env, struct lod_object *lo,
extent = &lod_comp->llc_extent;
QOS_DEBUG("comp[%d] %lld "DEXT"\n", i, size, PEXT(extent));
if (!lo->ldo_is_composite || size >= extent->e_start) {
- rc = lod_qos_prep_create(env, lo, attr, th, i);
+ rc = lod_qos_prep_create(env, lo, attr, th, i
, 0
);
if (rc)
break;
}