X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Flod%2Flod_lov.c;h=693d94b4c35bfe5ea0e66be23c07e5593ee42447;hb=c8a04503ebca2e596ff75f6f7162e0df7ea1891e;hp=fefa3c72bd79d1e0e07890421ca54e0e9e9f2c11;hpb=725f3f8eb96203b5679da46bfc4e6d22bf1cdf9c;p=fs%2Flustre-release.git diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index fefa3c7..693d94b 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -23,7 +23,7 @@ * Copyright 2009 Sun Microsystems, Inc. All rights reserved * Use is subject to license terms. * - * Copyright (c) 2012, Intel Corporation. + * Copyright (c) 2012, 2013, Intel Corporation. */ /* * lustre/lod/lod_lov.c @@ -31,13 +31,11 @@ * Author: Alex Zhuravlev */ -#ifndef EXPORT_SYMTAB -# define EXPORT_SYMTAB -#endif #define DEBUG_SUBSYSTEM S_MDS #include #include +#include #include "lod_internal.h" @@ -187,6 +185,7 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, struct lod_tgt_desc *tgt_desc; struct lod_tgt_descs *ltd; struct obd_uuid obd_uuid; + bool for_ost; ENTRY; CDEBUG(D_CONFIG, "osp:%s idx:%d gen:%d\n", osp, index, gen); @@ -215,6 +214,7 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, data->ocd_index = index; if (strcmp(LUSTRE_OSC_NAME, type) == 0) { + for_ost = true; data->ocd_connect_flags |= OBD_CONNECT_AT | OBD_CONNECT_FULL20 | OBD_CONNECT_INDEX | @@ -228,13 +228,15 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, OBD_CONNECT_FID | OBD_CONNECT_LVB_TYPE | OBD_CONNECT_VERSION | - OBD_CONNECT_PINGLESS; + OBD_CONNECT_PINGLESS | + OBD_CONNECT_LFSCK; data->ocd_group = tgt_index; ltd = &lod->lod_ost_descs; } else { struct obd_import *imp = obd->u.cli.cl_import; + for_ost = false; data->ocd_ibits_known = MDS_INODELOCK_UPDATE; data->ocd_connect_flags |= OBD_CONNECT_ACL | OBD_CONNECT_MDS_CAPA | @@ -243,14 +245,15 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, OBD_CONNECT_MDS_MDS | OBD_CONNECT_FID | OBD_CONNECT_AT | - OBD_CONNECT_FULL20; + OBD_CONNECT_FULL20 | + OBD_CONNECT_LFSCK; /* XXX set MDS-MDS flags, remove this when running this * on client*/ data->ocd_connect_flags |= OBD_CONNECT_MDS_MDS; spin_lock(&imp->imp_lock); imp->imp_server_timeout = 1; spin_unlock(&imp->imp_lock); - imp->imp_client->cli_request_portal = MDS_MDS_PORTAL; + imp->imp_client->cli_request_portal = OUT_PORTAL; CDEBUG(D_OTHER, "%s: Set 'mds' portal and timeout\n", obd->obd_name); ltd = &lod->lod_mdt_descs; @@ -350,7 +353,12 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, if (lod->lod_recovery_completed) ldev->ld_ops->ldo_recovery_complete(env, ldev); - RETURN(0); + rc = lfsck_add_target(env, lod->lod_child, d, exp, index, for_ost); + if (rc != 0) + CERROR("Fail to add LFSCK target: name = %s, type = %s, " + "index = %u, rc = %d\n", osp, type, index, rc); + + RETURN(rc); out_pool: lod_ost_pool_remove(&lod->lod_pool_info, index); @@ -368,17 +376,23 @@ out_free: /* * helper function to schedule OST removal from the device table */ -static void __lod_del_device(struct lod_tgt_descs *ltd, - unsigned idx) +static void __lod_del_device(const struct lu_env *env, struct lod_device *lod, + struct lod_tgt_descs *ltd, unsigned idx, + bool for_ost) { LASSERT(LTD_TGT(ltd, idx)); + + lfsck_del_target(env, lod->lod_child, LTD_TGT(ltd, idx)->ltd_tgt, + idx, for_ost); + if (LTD_TGT(ltd, idx)->ltd_reap == 0) { LTD_TGT(ltd, idx)->ltd_reap = 1; ltd->ltd_death_row++; } } -int lod_fini_tgt(struct lod_device *lod, struct lod_tgt_descs *ltd) +int lod_fini_tgt(const struct lu_env *env, struct lod_device *lod, + struct lod_tgt_descs *ltd, bool for_ost) { int idx; @@ -387,7 +401,7 @@ int lod_fini_tgt(struct lod_device *lod, struct lod_tgt_descs *ltd) lod_getref(ltd); mutex_lock(<d->ltd_mutex); cfs_foreach_bit(ltd->ltd_tgt_bitmap, idx) - __lod_del_device(ltd, idx); + __lod_del_device(env, lod, ltd, idx, for_ost); mutex_unlock(<d->ltd_mutex); lod_putref(lod, ltd); CFS_FREE_BITMAP(ltd->ltd_tgt_bitmap); @@ -414,7 +428,7 @@ int lod_fini_tgt(struct lod_device *lod, struct lod_tgt_descs *ltd) */ int lod_del_device(const struct lu_env *env, struct lod_device *lod, struct lod_tgt_descs *ltd, char *osp, unsigned idx, - unsigned gen) + unsigned gen, bool for_ost) { struct obd_device *obd; int rc = 0; @@ -457,7 +471,7 @@ int lod_del_device(const struct lu_env *env, struct lod_device *lod, GOTO(out, rc = -EINVAL); } - __lod_del_device(ltd, idx); + __lod_del_device(env, lod, ltd, idx, for_ost); EXIT; out: mutex_unlock(<d->ltd_mutex); @@ -504,9 +518,8 @@ int lod_generate_and_set_lovea(const struct lu_env *env, ENTRY; LASSERT(lo); - LASSERT(lo->ldo_stripenr > 0); - magic = lo->ldo_pool ? LOV_MAGIC_V3 : LOV_MAGIC_V1; + magic = lo->ldo_pool != NULL ? LOV_MAGIC_V3 : LOV_MAGIC_V1; lmm_size = lov_mds_md_size(lo->ldo_stripenr, magic); if (info->lti_ea_store_size < lmm_size) { rc = lod_ea_store_resize(info, lmm_size); @@ -514,14 +527,19 @@ int lod_generate_and_set_lovea(const struct lu_env *env, RETURN(rc); } + if (lo->ldo_pattern == 0) /* default striping */ + lo->ldo_pattern = LOV_PATTERN_RAID0; + lmm = info->lti_ea_store; lmm->lmm_magic = cpu_to_le32(magic); - lmm->lmm_pattern = cpu_to_le32(LOV_PATTERN_RAID0); + lmm->lmm_pattern = cpu_to_le32(lo->ldo_pattern); fid_to_lmm_oi(fid, &lmm->lmm_oi); lmm_oi_cpu_to_le(&lmm->lmm_oi, &lmm->lmm_oi); lmm->lmm_stripe_size = cpu_to_le32(lo->ldo_stripe_size); lmm->lmm_stripe_count = cpu_to_le16(lo->ldo_stripenr); + if (lo->ldo_pattern & LOV_PATTERN_F_RELEASED) + lmm->lmm_stripe_count = cpu_to_le16(lo->ldo_released_stripenr); lmm->lmm_layout_gen = 0; if (magic == LOV_MAGIC_V1) { objs = &lmm->lmm_objects[0]; @@ -550,6 +568,7 @@ int lod_generate_and_set_lovea(const struct lu_env *env, objs[i].l_ost_gen = cpu_to_le32(0); rc = lod_fld_lookup(env, lod, fid, &index, LU_SEQ_RANGE_OST); if (rc < 0) { + lod_object_free_striping(env, lo); CERROR("%s: Can not locate "DFID": rc = %d\n", lod2obd(lod)->obd_name, PFID(fid), rc); RETURN(rc); @@ -662,6 +681,30 @@ int lod_store_def_striping(const struct lu_env *env, struct dt_object *dt, RETURN(rc); } +static int validate_lod_and_idx(struct lod_device *md, int idx) +{ + if (unlikely(idx >= md->lod_ost_descs.ltd_tgts_size || + !cfs_bitmap_check(md->lod_ost_bitmap, idx))) { + CERROR("%s: bad idx: %d of %d\n", lod2obd(md)->obd_name, idx, + md->lod_ost_descs.ltd_tgts_size); + return -EINVAL; + } + + if (unlikely(OST_TGT(md, idx) == NULL)) { + CERROR("%s: bad lod_tgt_desc for idx: %d\n", + lod2obd(md)->obd_name, idx); + return -EINVAL; + } + + if (unlikely(OST_TGT(md, idx)->ltd_ost == NULL)) { + CERROR("%s: invalid lod device, for idx: %d\n", + lod2obd(md)->obd_name , idx); + return -EINVAL; + } + + return 0; +} + /* * allocate array of objects pointers, find/create objects * stripenr and other fields should be initialized by this moment @@ -670,7 +713,7 @@ int lod_initialize_objects(const struct lu_env *env, struct lod_object *lo, struct lov_ost_data_v1 *objs) { struct lod_thread_info *info = lod_env_info(env); - struct lod_device *md = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); + struct lod_device *md; struct lu_object *o, *n; struct lu_device *nd; struct dt_object **stripe; @@ -678,7 +721,8 @@ int lod_initialize_objects(const struct lu_env *env, struct lod_object *lo, int i, idx, rc = 0; ENTRY; - LASSERT(lo); + LASSERT(lo != NULL); + md = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); LASSERT(lo->ldo_stripe == NULL); LASSERT(lo->ldo_stripenr > 0); LASSERT(lo->ldo_stripe_size > 0); @@ -690,22 +734,20 @@ int lod_initialize_objects(const struct lu_env *env, struct lod_object *lo, for (i = 0; i < lo->ldo_stripenr; i++) { ostid_le_to_cpu(&objs[i].l_ost_oi, &info->lti_ostid); - idx = le64_to_cpu(objs[i].l_ost_idx); + idx = le32_to_cpu(objs[i].l_ost_idx); rc = ostid_to_fid(&info->lti_fid, &info->lti_ostid, idx); if (rc != 0) GOTO(out, rc); LASSERTF(fid_is_sane(&info->lti_fid), ""DFID" insane!\n", PFID(&info->lti_fid)); - /* - * XXX: assertion is left for testing, to make - * sure we never process requests till configuration - * is completed. to be changed to -EINVAL - */ - lod_getref(&md->lod_ost_descs); - LASSERT(cfs_bitmap_check(md->lod_ost_bitmap, idx)); - LASSERT(OST_TGT(md,idx)); - LASSERTF(OST_TGT(md,idx)->ltd_ost, "idx %d\n", idx); + + rc = validate_lod_and_idx(md, idx); + if (unlikely(rc != 0)) { + lod_putref(md, &md->lod_ost_descs); + GOTO(out, rc); + } + nd = &OST_TGT(md,idx)->ltd_ost->dd_lu_dev; lod_putref(md, &md->lod_ost_descs); @@ -746,6 +788,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, struct lov_mds_md_v1 *lmm; struct lov_ost_data_v1 *objs; __u32 magic; + __u32 pattern; int rc = 0; ENTRY; @@ -755,15 +798,20 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, lmm = (struct lov_mds_md_v1 *) buf->lb_buf; magic = le32_to_cpu(lmm->lmm_magic); + pattern = le32_to_cpu(lmm->lmm_pattern); if (magic != LOV_MAGIC_V1 && magic != LOV_MAGIC_V3) GOTO(out, rc = -EINVAL); - if (le32_to_cpu(lmm->lmm_pattern) != LOV_PATTERN_RAID0) + if (lov_pattern(pattern) != LOV_PATTERN_RAID0) GOTO(out, rc = -EINVAL); + lo->ldo_pattern = pattern; lo->ldo_stripe_size = le32_to_cpu(lmm->lmm_stripe_size); - lo->ldo_stripenr = le16_to_cpu(lmm->lmm_stripe_count); lo->ldo_layout_gen = le16_to_cpu(lmm->lmm_layout_gen); + lo->ldo_stripenr = le16_to_cpu(lmm->lmm_stripe_count); + /* released file stripenr fixup. */ + if (pattern & LOV_PATTERN_F_RELEASED) + lo->ldo_stripenr = 0; LASSERT(buf->lb_len >= lov_mds_md_size(lo->ldo_stripenr, magic)); @@ -775,7 +823,8 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, objs = &lmm->lmm_objects[0]; } - rc = lod_initialize_objects(env, lo, objs); + if (lo->ldo_stripenr > 0) + rc = lod_initialize_objects(env, lo, objs); out: RETURN(rc); @@ -798,13 +847,8 @@ int lod_load_striping(const struct lu_env *env, struct lod_object *lo) */ dt_write_lock(env, next, 0); /* already initialized? */ - if (lo->ldo_stripe) { - int i; - /* check validity */ - for (i = 0; i < lo->ldo_stripenr; i++) - LASSERTF(lo->ldo_stripe[i], "stripe %d is NULL\n", i); + if (lo->ldo_stripe != NULL) GOTO(out, rc = 0); - } if (!dt_object_exists(next)) GOTO(out, rc = 0); @@ -813,8 +857,6 @@ int lod_load_striping(const struct lu_env *env, struct lod_object *lo) if (!(lu_object_attr(lod2lu_obj(lo)) & S_IFREG)) GOTO(out, rc = 0); - LASSERT(lo->ldo_stripenr == 0); - rc = lod_get_lov_ea(env, lo); if (rc <= 0) GOTO(out, rc); @@ -835,94 +877,130 @@ int lod_verify_striping(struct lod_device *d, const struct lu_buf *buf, int specific) { struct lov_user_md_v1 *lum; - struct lov_user_md_v3 *v3 = NULL; + struct lov_user_md_v3 *lum3; struct pool_desc *pool = NULL; - int rc; + __u32 magic; + __u32 stripe_size; + __u16 stripe_count; + __u16 stripe_offset; + size_t lum_size; + int rc = 0; ENTRY; lum = buf->lb_buf; - if (lum->lmm_magic != LOV_USER_MAGIC_V1 && - lum->lmm_magic != LOV_USER_MAGIC_V3 && - lum->lmm_magic != LOV_MAGIC_V1_DEF && - lum->lmm_magic != LOV_MAGIC_V3_DEF) { - CDEBUG(D_IOCTL, "bad userland LOV MAGIC: %#x\n", - lum->lmm_magic); - RETURN(-EINVAL); + LASSERT(sizeof(*lum) < sizeof(*lum3)); + + if (buf->lb_len < sizeof(*lum)) { + CDEBUG(D_IOCTL, "buf len %zd too small for lov_user_md\n", + buf->lb_len); + GOTO(out, rc = -EINVAL); } - if ((specific && lum->lmm_pattern != LOV_PATTERN_RAID0) || - (specific == 0 && lum->lmm_pattern != 0)) { + magic = le32_to_cpu(lum->lmm_magic); + if (magic != LOV_USER_MAGIC_V1 && + magic != LOV_USER_MAGIC_V3 && + magic != LOV_MAGIC_V1_DEF && + magic != LOV_MAGIC_V3_DEF) { + CDEBUG(D_IOCTL, "bad userland LOV MAGIC: %#x\n", magic); + GOTO(out, rc = -EINVAL); + } + + if ((specific && le32_to_cpu(lum->lmm_pattern) != LOV_PATTERN_RAID0) || + (!specific && lum->lmm_pattern != 0)) { CDEBUG(D_IOCTL, "bad userland stripe pattern: %#x\n", - lum->lmm_pattern); - RETURN(-EINVAL); + le32_to_cpu(lum->lmm_pattern)); + GOTO(out, rc = -EINVAL); } /* 64kB is the largest common page size we see (ia64), and matches the * check in lfs */ - if (lum->lmm_stripe_size & (LOV_MIN_STRIPE_SIZE - 1)) { - CDEBUG(D_IOCTL, "stripe size %u not multiple of %u, fixing\n", - lum->lmm_stripe_size, LOV_MIN_STRIPE_SIZE); - RETURN(-EINVAL); + stripe_size = le32_to_cpu(lum->lmm_stripe_size); + if (stripe_size & (LOV_MIN_STRIPE_SIZE - 1)) { + CDEBUG(D_IOCTL, "stripe size %u not a multiple of %u\n", + stripe_size, LOV_MIN_STRIPE_SIZE); + GOTO(out, rc = -EINVAL); } /* an offset of -1 is treated as a "special" valid offset */ - if (lum->lmm_stripe_offset != (typeof(lum->lmm_stripe_offset))(-1)) { + stripe_offset = le16_to_cpu(lum->lmm_stripe_offset); + if (stripe_offset != (typeof(stripe_offset))-1) { /* if offset is not within valid range [0, osts_size) */ - if (lum->lmm_stripe_offset >= d->lod_osts_size) { + if (stripe_offset >= d->lod_osts_size) { CDEBUG(D_IOCTL, "stripe offset %u >= bitmap size %u\n", - lum->lmm_stripe_offset, d->lod_osts_size); - RETURN(-EINVAL); + stripe_offset, d->lod_osts_size); + GOTO(out, rc = -EINVAL); } /* if lmm_stripe_offset is *not* in bitmap */ - if (!cfs_bitmap_check(d->lod_ost_bitmap, - lum->lmm_stripe_offset)) { + if (!cfs_bitmap_check(d->lod_ost_bitmap, stripe_offset)) { CDEBUG(D_IOCTL, "stripe offset %u not in bitmap\n", - lum->lmm_stripe_offset); - RETURN(-EINVAL); + stripe_offset); + GOTO(out, rc = -EINVAL); } } - if (lum->lmm_magic == LOV_USER_MAGIC_V3) - v3 = buf->lb_buf; + stripe_count = le16_to_cpu(lum->lmm_stripe_count); + if (magic == LOV_USER_MAGIC_V1 || magic == LOV_MAGIC_V1_DEF) + lum_size = offsetof(struct lov_user_md_v1, + lmm_objects[stripe_count]); + else if (magic == LOV_USER_MAGIC_V3 || magic == LOV_MAGIC_V3_DEF) + lum_size = offsetof(struct lov_user_md_v3, + lmm_objects[stripe_count]); + else + LBUG(); + + if (specific && buf->lb_len != lum_size) { + CDEBUG(D_IOCTL, "invalid buf len %zd for lov_user_md with " + "magic %#x and stripe_count %u\n", + buf->lb_len, magic, stripe_count); + GOTO(out, rc = -EINVAL); + } - if (v3) - /* In the function below, .hs_keycmp resolves to - * pool_hashkey_keycmp() */ - /* coverity[overrun-buffer-val] */ - pool = lod_find_pool(d, v3->lmm_pool_name); + if (!(magic == LOV_USER_MAGIC_V3 || magic == LOV_MAGIC_V3_DEF)) + goto out; - if (pool != NULL) { - __u16 offs = v3->lmm_stripe_offset; + lum3 = buf->lb_buf; + if (buf->lb_len < sizeof(*lum3)) { + CDEBUG(D_IOCTL, "buf len %zd too small for lov_user_md_v3\n", + buf->lb_len); + GOTO(out, rc = -EINVAL); + } - if (offs != (typeof(v3->lmm_stripe_offset))(-1)) { - rc = lod_check_index_in_pool(offs, pool); - if (rc < 0) { - lod_pool_putref(pool); - RETURN(-EINVAL); - } - } + /* In the function below, .hs_keycmp resolves to + * pool_hashkey_keycmp() */ + /* coverity[overrun-buffer-val] */ + pool = lod_find_pool(d, lum3->lmm_pool_name); + if (pool == NULL) + goto out; + + if (stripe_offset != (typeof(stripe_offset))-1) { + rc = lod_check_index_in_pool(stripe_offset, pool); + if (rc < 0) + GOTO(out, rc = -EINVAL); + } - if (specific && lum->lmm_stripe_count > pool_tgt_count(pool)) { - CDEBUG(D_IOCTL, - "stripe count %u > # OSTs %u in the pool\n", - lum->lmm_stripe_count, pool_tgt_count(pool)); - lod_pool_putref(pool); - RETURN(-EINVAL); - } + if (specific && stripe_count > pool_tgt_count(pool)) { + CDEBUG(D_IOCTL, + "stripe count %u > # OSTs %u in the pool\n", + stripe_count, pool_tgt_count(pool)); + GOTO(out, rc = -EINVAL); + } +out: + if (pool != NULL) lod_pool_putref(pool); - } - RETURN(0); + RETURN(rc); } void lod_fix_desc_stripe_size(__u64 *val) { - if (*val < LOV_DEFAULT_STRIPE_SIZE) { - LCONSOLE_WARN("Increasing default stripe size to min %u\n", - LOV_DEFAULT_STRIPE_SIZE); + if (*val < LOV_MIN_STRIPE_SIZE) { + if (*val != 0) + LCONSOLE_INFO("Increasing default stripe size to " + "minimum value %u\n", + LOV_DEFAULT_STRIPE_SIZE); *val = LOV_DEFAULT_STRIPE_SIZE; } else if (*val & (LOV_MIN_STRIPE_SIZE - 1)) { *val &= ~(LOV_MIN_STRIPE_SIZE - 1); @@ -1015,11 +1093,6 @@ int lod_pools_init(struct lod_device *lod, struct lustre_cfg *lcfg) lod->lod_qos.lq_prio_free = 232; /* Default threshold for rr (roughly 17%) */ lod->lod_qos.lq_threshold_rr = 43; - /* Init statfs fields */ - OBD_ALLOC_PTR(lod->lod_qos.lq_statfs_data); - if (NULL == lod->lod_qos.lq_statfs_data) - RETURN(-ENOMEM); - cfs_waitq_init(&lod->lod_qos.lq_statfs_waitq); /* Set up OST pool environment */ lod->lod_pools_hash_body = cfs_hash_create("POOLS", HASH_POOLS_CUR_BITS, @@ -1029,8 +1102,9 @@ int lod_pools_init(struct lod_device *lod, struct lustre_cfg *lcfg) CFS_HASH_MAX_THETA, &pool_hash_operations, CFS_HASH_DEFAULT); - if (!lod->lod_pools_hash_body) - GOTO(out_statfs, rc = -ENOMEM); + if (lod->lod_pools_hash_body == NULL) + RETURN(-ENOMEM); + CFS_INIT_LIST_HEAD(&lod->lod_pool_list); lod->lod_pool_count = 0; rc = lod_ost_pool_init(&lod->lod_pool_info, 0); @@ -1046,8 +1120,7 @@ out_pool_info: lod_ost_pool_free(&lod->lod_pool_info); out_hash: cfs_hash_putref(lod->lod_pools_hash_body); -out_statfs: - OBD_FREE_PTR(lod->lod_qos.lq_statfs_data); + return rc; } @@ -1071,7 +1144,7 @@ int lod_pools_fini(struct lod_device *lod) cfs_hash_putref(lod->lod_pools_hash_body); lod_ost_pool_free(&(lod->lod_qos.lq_rr.lqr_pool)); lod_ost_pool_free(&lod->lod_pool_info); - OBD_FREE_PTR(lod->lod_qos.lq_statfs_data); + RETURN(0); }