From 087ec276de0bc79db66b614e1f17ff0bf631464f Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Sat, 16 Dec 2017 16:22:10 +0800 Subject: [PATCH] LU-8616 dne: allow mkdir with specific MDTs Silimiar to create file on specific OSTs, allow 'lfs mkdir' to mkdir on specific MDTs. This is achieved by allowing 'lfs mkdir -i' specify multiple MDTs, and adding LMV_USER_MAGIC_SPECIFIC. Signed-off-by: Lai Siyao Change-Id: I3876707103465d1659afc80914ed6f9b58da25eb Reviewed-on: https://review.whamcloud.com/30566 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Fan Yong --- lustre/include/lustre/lustreapi.h | 7 + lustre/include/uapi/linux/lustre/lustre_user.h | 1 + lustre/llite/dir.c | 17 +- lustre/lod/lod_object.c | 79 ++++-- lustre/mdd/mdd_dir.c | 3 +- lustre/ptlrpc/pack_generic.c | 22 ++ lustre/tests/sanity.sh | 20 ++ lustre/utils/lfs.c | 192 +++++++------- lustre/utils/liblustreapi.c | 330 +++++++++++++++++++------ lustre/utils/lustreapi_internal.h | 15 ++ 10 files changed, 489 insertions(+), 197 deletions(-) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 80ba7aa..59253dd 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -117,6 +117,8 @@ struct llapi_stripe_param { __u32 lsp_osts[0]; }; +#define lsp_tgts lsp_osts + int llapi_file_open_param(const char *name, int flags, mode_t mode, const struct llapi_stripe_param *param); int llapi_file_create(const char *name, unsigned long long stripe_size, @@ -288,9 +290,13 @@ int llapi_getstripe(char *path, struct find_param *param); int llapi_find(char *path, struct find_param *param); int llapi_file_fget_mdtidx(int fd, int *mdtidx); +int llapi_dir_set_default_lmv(const char *name, + const struct llapi_stripe_param *param); int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, int stripe_count, int stripe_pattern, const char *pool_name); +int llapi_dir_create_param(const char *name, mode_t mode, + const struct llapi_stripe_param *param); int llapi_dir_create_pool(const char *name, int flags, int stripe_offset, int stripe_count, int stripe_pattern, const char *poolname); @@ -308,6 +314,7 @@ int llapi_file_fget_lov_uuid(int fd, struct obd_uuid *lov_uuid); int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count); int llapi_lmv_get_uuids(int fd, struct obd_uuid *uuidp, int *mdt_count); int llapi_is_lustre_mnttype(const char *type); +int llapi_search_tgt(char *fsname, char *poolname, char *tgtname, bool is_mdt); int llapi_search_ost(char *fsname, char *poolname, char *ostname); int llapi_get_obd_count(char *mnt, int *count, int is_mdt); int llapi_parse_size(const char *optarg, unsigned long long *size, diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index a07d74c..aaed47e 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -469,6 +469,7 @@ struct fsxattr { #define LMV_USER_MAGIC 0x0CD30CD0 /* default lmv magic */ #define LMV_USER_MAGIC_V0 0x0CD20CD0 /* old default lmv magic*/ +#define LMV_USER_MAGIC_SPECIFIC 0x0CD40CD0 #define LOV_PATTERN_NONE 0x000 #define LOV_PATTERN_RAID0 0x001 diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index f174865..5c8f311 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -434,7 +434,7 @@ static int ll_send_mgc_param(struct obd_export *mgc, char *string) * <0 if the creation is failed. */ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, - const char *dirname, umode_t mode) + size_t len, const char *dirname, umode_t mode) { struct inode *parent = dparent->d_inode; struct ptlrpc_request *request = NULL; @@ -453,7 +453,8 @@ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, int err; ENTRY; - if (unlikely(lump->lum_magic != LMV_USER_MAGIC)) + if (unlikely(lump->lum_magic != LMV_USER_MAGIC && + lump->lum_magic != LMV_USER_MAGIC_SPECIFIC)) RETURN(-EINVAL); CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) name %s " @@ -469,7 +470,8 @@ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, !OBD_FAIL_CHECK(OBD_FAIL_LLITE_NO_CHECK_DEAD)) RETURN(-ENOENT); - if (lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC)) + if (lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC) && + lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC_SPECIFIC)) lustre_swab_lmv_user_md(lump); if (!IS_POSIXACL(parent) || !exp_connect_umask(ll_i2mdexp(parent))) @@ -494,7 +496,7 @@ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, } op_data->op_cli_flags |= CLI_SET_MEA; - err = md_create(sbi->ll_md_exp, op_data, lump, sizeof(*lump), mode, + err = md_create(sbi->ll_md_exp, op_data, lump, len, mode, from_kuid(&init_user_ns, current_fsuid()), from_kgid(&init_user_ns, current_fsgid()), cfs_curproc_cap_pack(), 0, &request); @@ -1247,8 +1249,9 @@ out_free: lum = (struct lmv_user_md *)data->ioc_inlbuf2; lumlen = data->ioc_inllen2; - if (lum->lum_magic != LMV_USER_MAGIC || - lumlen != sizeof(*lum)) { + if ((lum->lum_magic != LMV_USER_MAGIC && + lum->lum_magic != LMV_USER_MAGIC_SPECIFIC) || + lumlen < sizeof(*lum)) { CERROR("%s: wrong lum magic %x or size %d: rc = %d\n", filename, lum->lum_magic, lumlen, -EFAULT); GOTO(lmv_out_free, rc = -EINVAL); @@ -1259,7 +1262,7 @@ out_free: #else mode = data->ioc_type; #endif - rc = ll_dir_setdirstripe(dentry, lum, filename, mode); + rc = ll_dir_setdirstripe(dentry, lum, lumlen, filename, mode); lmv_out_free: OBD_FREE_LARGE(buf, len); RETURN(rc); diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index bf56b50..e7d9dbc 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -1859,10 +1859,12 @@ static int lod_prep_md_striped_create(const struct lu_env *env, int rc = 0; __u32 i; __u32 j; + bool is_specific = false; ENTRY; /* The lum has been verifed in lod_verify_md_striping */ - LASSERT(le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC); + LASSERT(le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC || + le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC_SPECIFIC); LASSERT(le32_to_cpu(lum->lum_stripe_count) > 0); stripe_count = le32_to_cpu(lum->lum_stripe_count); @@ -1878,6 +1880,12 @@ static int lod_prep_md_striped_create(const struct lu_env *env, /* Start index must be the master MDT */ master_index = lu_site2seq(lod2lu_dev(lod)->ld_site)->ss_node_id; idx_array[0] = master_index; + if (le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC_SPECIFIC) { + is_specific = true; + for (i = 1; i < stripe_count; i++) + idx_array[i] = le32_to_cpu(lum->lum_objects[i].lum_mds); + } + for (i = 0; i < stripe_count; i++) { struct lod_tgt_desc *tgt = NULL; struct dt_object *dto; @@ -1896,7 +1904,8 @@ static int lod_prep_md_striped_create(const struct lu_env *env, CDEBUG(D_INFO, "try idx %d, mdt cnt %u, allocated %u\n", idx, lod->lod_remote_mdt_count + 1, i); - if (likely(!OBD_FAIL_CHECK(OBD_FAIL_LARGE_STRIPE))) { + if (likely(!is_specific && + !OBD_FAIL_CHECK(OBD_FAIL_LARGE_STRIPE))) { /* check whether the idx already exists * in current allocated array */ for (k = 0; k < i; k++) { @@ -1959,7 +1968,7 @@ static int lod_prep_md_striped_create(const struct lu_env *env, idx, i, PFID(&fid)); idx_array[i] = idx; /* Set the start index for next stripe allocation */ - if (i < stripe_count - 1) + if (!is_specific && i < stripe_count - 1) idx_array[i + 1] = (idx + 1) % (lod->lod_remote_mdt_count + 1); /* tgt_dt and fid must be ready after search avaible OSP @@ -2042,7 +2051,7 @@ static int lod_declare_xattr_set_lmv(const struct lu_env *env, le32_to_cpu(lum->lum_magic), le32_to_cpu(lum->lum_stripe_count), (int)le32_to_cpu(lum->lum_stripe_offset)); - if (le32_to_cpu(lum->lum_stripe_count) == 0) + if (lo->ldo_dir_stripe_count == 0) GOTO(out, rc = 0); /* prepare dir striped objects */ @@ -3357,6 +3366,7 @@ out: * \param[in] env execution environment * \param[in] dt object * \param[in] attr attributes the stripes will be created with + * \param[in] lmu lmv_user_md if MDT indices are specified * \param[in] dof format of stripes (see OSD API description) * \param[in] th transaction handle * \param[in] declare where to call "declare" or "execute" methods @@ -3367,6 +3377,7 @@ out: static int lod_dir_striping_create_internal(const struct lu_env *env, struct dt_object *dt, struct lu_attr *attr, + const struct lu_buf *lmu, struct dt_object_format *dof, struct thandle *th, bool declare) @@ -3383,31 +3394,34 @@ static int lod_dir_striping_create_internal(const struct lu_env *env, if (!LMVEA_DELETE_VALUES(lo->ldo_dir_stripe_count, lo->ldo_dir_stripe_offset)) { - struct lmv_user_md_v1 *v1 = info->lti_ea_store; - int stripe_count = lo->ldo_dir_stripe_count; + if (!lmu) { + struct lmv_user_md_v1 *v1 = info->lti_ea_store; + int stripe_count = lo->ldo_dir_stripe_count; - if (info->lti_ea_store_size < sizeof(*v1)) { - rc = lod_ea_store_resize(info, sizeof(*v1)); - if (rc != 0) - RETURN(rc); - v1 = info->lti_ea_store; - } + if (info->lti_ea_store_size < sizeof(*v1)) { + rc = lod_ea_store_resize(info, sizeof(*v1)); + if (rc != 0) + RETURN(rc); + v1 = info->lti_ea_store; + } - memset(v1, 0, sizeof(*v1)); - v1->lum_magic = cpu_to_le32(LMV_USER_MAGIC); - v1->lum_stripe_count = cpu_to_le32(stripe_count); - v1->lum_stripe_offset = - cpu_to_le32(lo->ldo_dir_stripe_offset); + memset(v1, 0, sizeof(*v1)); + v1->lum_magic = cpu_to_le32(LMV_USER_MAGIC); + v1->lum_stripe_count = cpu_to_le32(stripe_count); + v1->lum_stripe_offset = + cpu_to_le32(lo->ldo_dir_stripe_offset); - info->lti_buf.lb_buf = v1; - info->lti_buf.lb_len = sizeof(*v1); + info->lti_buf.lb_buf = v1; + info->lti_buf.lb_len = sizeof(*v1); + lmu = &info->lti_buf; + } if (declare) - rc = lod_declare_xattr_set_lmv(env, dt, attr, - &info->lti_buf, dof, th); + rc = lod_declare_xattr_set_lmv(env, dt, attr, lmu, dof, + th); else - rc = lod_xattr_set_lmv(env, dt, &info->lti_buf, - XATTR_NAME_LMV, 0, th); + rc = lod_xattr_set_lmv(env, dt, lmu, XATTR_NAME_LMV, 0, + th); if (rc != 0) RETURN(rc); } @@ -3485,10 +3499,12 @@ static int lod_dir_striping_create_internal(const struct lu_env *env, static int lod_declare_dir_striping_create(const struct lu_env *env, struct dt_object *dt, struct lu_attr *attr, + struct lu_buf *lmu, struct dt_object_format *dof, struct thandle *th) { - return lod_dir_striping_create_internal(env, dt, attr, dof, th, true); + return lod_dir_striping_create_internal(env, dt, attr, lmu, dof, th, + true); } static int lod_dir_striping_create(const struct lu_env *env, @@ -3497,7 +3513,8 @@ static int lod_dir_striping_create(const struct lu_env *env, struct dt_object_format *dof, struct thandle *th) { - return lod_dir_striping_create_internal(env, dt, attr, dof, th, false); + return lod_dir_striping_create_internal(env, dt, attr, NULL, dof, th, + false); } /** @@ -4220,7 +4237,8 @@ static void lod_ah_init(const struct lu_env *env, * stripe count and try to create dir by default stripe. */ if (ah->dah_eadata != NULL && ah->dah_eadata_len != 0 && - le32_to_cpu(lum1->lum_magic) == LMV_USER_MAGIC) { + (le32_to_cpu(lum1->lum_magic) == LMV_USER_MAGIC || + le32_to_cpu(lum1->lum_magic) == LMV_USER_MAGIC_SPECIFIC)) { lc->ldo_dir_stripe_count = le32_to_cpu(lum1->lum_stripe_count); lc->ldo_dir_stripe_offset = @@ -4556,6 +4574,8 @@ static int lod_declare_create(const struct lu_env *env, struct dt_object *dt, NULL, th); } else if (dof->dof_type == DFT_DIR) { struct seq_server_site *ss; + struct lu_buf buf = { NULL }; + struct lu_buf *lmu = NULL; ss = lu_site2seq(dt->do_lu.lo_dev->ld_site); @@ -4605,9 +4625,14 @@ static int lod_declare_create(const struct lu_env *env, struct dt_object *dt, else GOTO(out, rc = -EINVAL); } + } else if (hint && hint->dah_eadata) { + lmu = &buf; + lmu->lb_buf = (void *)hint->dah_eadata; + lmu->lb_len = hint->dah_eadata_len; } - rc = lod_declare_dir_striping_create(env, dt, attr, dof, th); + rc = lod_declare_dir_striping_create(env, dt, attr, lmu, dof, + th); } out: /* failed to create striping or to set initial size, let's reset diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index 7a49239..46d82d5 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -2137,7 +2137,8 @@ static int mdd_create_sanity_check(const struct lu_env *env, spec->u.sp_ea.eadata != NULL && spec->u.sp_ea.eadatalen > 0) { const struct lmv_user_md *lum = spec->u.sp_ea.eadata; - if (unlikely(le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC) && + if (le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC && + le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC_SPECIFIC && le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC_V0) { rc = -EINVAL; CERROR("%s: invalid lmv_user_md: magic = %x, " diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index c4f3cbb..0338893 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2120,14 +2120,36 @@ void lustre_swab_lmv_mds_md(union lmv_mds_md *lmm) } EXPORT_SYMBOL(lustre_swab_lmv_mds_md); +void lustre_swab_lmv_user_md_objects(struct lmv_user_mds_data *lmd, + int stripe_count) +{ + int i; + + for (i = 0; i < stripe_count; i++) + __swab32s(&(lmd[i].lum_mds)); +} +EXPORT_SYMBOL(lustre_swab_lmv_user_md_objects); + + void lustre_swab_lmv_user_md(struct lmv_user_md *lum) { + __u32 count = lum->lum_stripe_count; + __swab32s(&lum->lum_magic); __swab32s(&lum->lum_stripe_count); __swab32s(&lum->lum_stripe_offset); __swab32s(&lum->lum_hash_type); __swab32s(&lum->lum_type); CLASSERT(offsetof(typeof(*lum), lum_padding1) != 0); + switch (lum->lum_magic) { + case LMV_USER_MAGIC_SPECIFIC: + count = lum->lum_stripe_count; + case __swab32(LMV_USER_MAGIC_SPECIFIC): + lustre_swab_lmv_user_md_objects(lum->lum_objects, count); + break; + default: + break; + } } EXPORT_SYMBOL(lustre_swab_lmv_user_md); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 8cdf3fe..eb0ff0e 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -17402,6 +17402,26 @@ test_411() { } run_test 411 "Slab allocation error with cgroup does not LBUG" +test_412() { + [ $MDSCOUNT -lt 2 ] && + skip "We need at least 2 MDTs for this test" && return + + if [ $(lustre_version_code mds1) -lt $(version_code 2.10.55) ]; then + skip "Need server version at least 2.10.55" & exit 0 + fi + + $LFS mkdir -i $((MDSCOUNT - 1)),$((MDSCOUNT - 2)) $DIR/$tdir || + error "mkdir failed" + $LFS getdirstripe $DIR/$tdir + stripe_index=$($LFS getdirstripe -i $DIR/$tdir) + [ $stripe_index -eq $((MDSCOUNT - 1)) ] || + error "expect $((MDSCOUT - 1)) get $stripe_index" + stripe_count=$($LFS getdirstripe -T $DIR/$tdir) + [ $stripe_count -eq 2 ] || + error "expect 2 get $stripe_count" +} +run_test 412 "mkdir on specific MDTs" + prep_801() { [[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] || [[ $(lustre_version_code ost1) -lt $(version_code 2.9.55) ]] && diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 9f168aa..a5c87ef 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -1387,21 +1387,21 @@ static int mirror_extend(char *fname, struct mirror_args *mirror_list, } /** - * Parse a string containing an OST index list into an array of integers. + * Parse a string containing an target index list into an array of integers. * * The input string contains a comma delimited list of individual * indices and ranges, for example "1,2-4,7". Add the indices into the - * \a osts array and remove duplicates. + * \a tgts array and remove duplicates. * - * \param[out] osts array to store indices in - * \param[in] size size of \a osts array - * \param[in] offset starting index in \a osts + * \param[out] tgts array to store indices in + * \param[in] size size of \a tgts array + * \param[in] offset starting index in \a tgts * \param[in] arg string containing OST index list * - * \retval positive number of indices in \a osts + * \retval positive number of indices in \a tgts * \retval -EINVAL unable to parse \a arg */ -static int parse_targets(__u32 *osts, int size, int offset, char *arg) +static int parse_targets(__u32 *tgts, int size, int offset, char *arg) { int rc; int nr = offset; @@ -1431,8 +1431,6 @@ static int parse_targets(__u32 *osts, int size, int offset, char *arg) break; if (*endptr != '-' && *endptr != '\0') /* has invalid data */ break; - if (start_index < 0) - break; end_index = start_index; if (*endptr == '-') { @@ -1448,11 +1446,11 @@ static int parse_targets(__u32 *osts, int size, int offset, char *arg) /* remove duplicate */ for (j = 0; j < offset; j++) { - if (osts[j] == i) + if (tgts[j] == i) break; } if (j == offset) { /* no duplicate */ - osts[nr++] = i; + tgts[nr++] = i; --slots; } } @@ -1476,9 +1474,9 @@ struct lfs_setstripe_args { long long lsa_stripe_count; long long lsa_stripe_off; __u32 lsa_comp_flags; - int lsa_nr_osts; + int lsa_nr_tgts; unsigned long long lsa_pattern; - __u32 *lsa_osts; + __u32 *lsa_tgts; char *lsa_pool_name; }; @@ -1597,10 +1595,10 @@ static int comp_args_to_layout(struct llapi_layout **composite, lsa->lsa_stripe_size); return -EINVAL; } - if (lsa->lsa_nr_osts != 0) { + if (lsa->lsa_nr_tgts != 0) { fprintf(stderr, "Option 'ost-list' can't be specified " "with Data-on-MDT component: '%i'\n", - lsa->lsa_nr_osts); + lsa->lsa_nr_tgts); return -EINVAL; } if (lsa->lsa_stripe_off != LLAPI_LAYOUT_DEFAULT) { @@ -1656,18 +1654,18 @@ static int comp_args_to_layout(struct llapi_layout **composite, } } - if (lsa->lsa_nr_osts > 0) { + if (lsa->lsa_nr_tgts > 0) { if (lsa->lsa_stripe_count > 0 && lsa->lsa_stripe_count != LLAPI_LAYOUT_DEFAULT && lsa->lsa_stripe_count != LLAPI_LAYOUT_WIDE && - lsa->lsa_nr_osts != lsa->lsa_stripe_count) { + lsa->lsa_nr_tgts != lsa->lsa_stripe_count) { fprintf(stderr, "stripe_count(%lld) != nr_osts(%d)\n", - lsa->lsa_stripe_count, lsa->lsa_nr_osts); + lsa->lsa_stripe_count, lsa->lsa_nr_tgts); return -EINVAL; } - for (i = 0; i < lsa->lsa_nr_osts; i++) { + for (i = 0; i < lsa->lsa_nr_tgts; i++) { rc = llapi_layout_ost_index_set(layout, i, - lsa->lsa_osts[i]); + lsa->lsa_tgts[i]); if (rc) break; } @@ -2226,17 +2224,17 @@ static int lfs_setstripe0(int argc, char **argv, enum setstripe_origin opc) lpp = &last_mirror->m_layout; break; case 'o': - lsa.lsa_nr_osts = parse_targets(osts, + lsa.lsa_nr_tgts = parse_targets(osts, sizeof(osts) / sizeof(__u32), - lsa.lsa_nr_osts, optarg); - if (lsa.lsa_nr_osts < 0) { + lsa.lsa_nr_tgts, optarg); + if (lsa.lsa_nr_tgts < 0) { fprintf(stderr, "%s %s: invalid OST target(s) '%s'\n", progname, argv[0], optarg); goto usage_error; } - lsa.lsa_osts = osts; + lsa.lsa_tgts = osts; if (lsa.lsa_stripe_off == LLAPI_LAYOUT_DEFAULT) lsa.lsa_stripe_off = osts[0]; break; @@ -2420,7 +2418,7 @@ static int lfs_setstripe0(int argc, char **argv, enum setstripe_origin opc) } else if (layout == NULL) { /* initialize stripe parameters */ param = calloc(1, offsetof(typeof(*param), - lsp_osts[lsa.lsa_nr_osts])); + lsp_osts[lsa.lsa_nr_tgts])); if (param == NULL) { fprintf(stderr, "%s %s: cannot allocate memory for parameters: %s\n", @@ -2443,23 +2441,23 @@ static int lfs_setstripe0(int argc, char **argv, enum setstripe_origin opc) param->lsp_stripe_offset = lsa.lsa_stripe_off; param->lsp_pool = lsa.lsa_pool_name; param->lsp_is_specific = false; - if (lsa.lsa_nr_osts > 0) { + if (lsa.lsa_nr_tgts > 0) { if (lsa.lsa_stripe_count > 0 && lsa.lsa_stripe_count != LLAPI_LAYOUT_DEFAULT && lsa.lsa_stripe_count != LLAPI_LAYOUT_WIDE && - lsa.lsa_nr_osts != lsa.lsa_stripe_count) { + lsa.lsa_nr_tgts != lsa.lsa_stripe_count) { fprintf(stderr, "error: %s: stripe count %lld " "doesn't match the number of OSTs: %d\n" , argv[0], lsa.lsa_stripe_count, - lsa.lsa_nr_osts); + lsa.lsa_nr_tgts); free(param); goto usage_error; } param->lsp_is_specific = true; - param->lsp_stripe_count = lsa.lsa_nr_osts; + param->lsp_stripe_count = lsa.lsa_nr_tgts; memcpy(param->lsp_osts, osts, - sizeof(*osts) * lsa.lsa_nr_osts); + sizeof(*osts) * lsa.lsa_nr_tgts); } } @@ -3594,14 +3592,11 @@ static int lfs_setdirstripe(int argc, char **argv) { char *dname; int result; - unsigned int stripe_offset = -1; - unsigned int stripe_count = 1; - enum lmv_hash_type hash_type; + struct lfs_setstripe_args lsa; + struct llapi_stripe_param *param = NULL; + __u32 mdts[LMV_MAX_STRIPE_COUNT] = { 0 }; char *end; int c; - char *stripe_offset_opt = NULL; - char *stripe_count_opt = NULL; - char *stripe_hash_opt = NULL; char *mode_opt = NULL; bool default_stripe = false; mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO; @@ -3631,6 +3626,8 @@ static int lfs_setdirstripe(int argc, char **argv) { .val = 'D', .name = "default", .has_arg = no_argument }, { .name = NULL } }; + setstripe_args_init(&lsa); + while ((c = getopt_long(argc, argv, "c:dDi:H:m:t:", long_opts, NULL)) >= 0) { switch (c) { @@ -3644,7 +3641,13 @@ static int lfs_setdirstripe(int argc, char **argv) "%s %s: warning: '--count' deprecated, use '--mdt-count' instead\n", progname, argv[0]); #endif - stripe_count_opt = optarg; + lsa.lsa_stripe_count = strtoul(optarg, &end, 0); + if (*end != '\0') { + fprintf(stderr, + "%s %s: invalid stripe count '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } break; case 'd': delete = true; @@ -3660,7 +3663,19 @@ static int lfs_setdirstripe(int argc, char **argv) "%s %s: warning: '--index' deprecated, use '--mdt-index' instead\n", progname, argv[0]); #endif - stripe_offset_opt = optarg; + lsa.lsa_nr_tgts = parse_targets(mdts, + sizeof(mdts) / sizeof(__u32), + lsa.lsa_nr_tgts, optarg); + if (lsa.lsa_nr_tgts < 0) { + fprintf(stderr, + "%s %s: invalid MDT target(s) '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } + + lsa.lsa_tgts = mdts; + if (lsa.lsa_stripe_off == LLAPI_LAYOUT_DEFAULT) + lsa.lsa_stripe_off = mdts[0]; break; case 'm': mode_opt = optarg; @@ -3675,7 +3690,13 @@ static int lfs_setdirstripe(int argc, char **argv) "%s %s: warning: '--hash-type' deprecated, use '--mdt-hash' instead\n", progname, argv[0]); #endif - stripe_hash_opt = optarg; + lsa.lsa_pattern = check_hashtype(optarg); + if (lsa.lsa_pattern == 0) { + fprintf(stderr, + "%s %s: bad stripe hash type '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } break; default: fprintf(stderr, "%s %s: unrecognized option '%s'\n", @@ -3690,36 +3711,23 @@ static int lfs_setdirstripe(int argc, char **argv) return CMD_HELP; } - if (!delete && stripe_offset_opt == NULL && stripe_count_opt == NULL) { + if (!delete && lsa.lsa_stripe_off == LLAPI_LAYOUT_DEFAULT && + lsa.lsa_stripe_count == LLAPI_LAYOUT_DEFAULT) { fprintf(stderr, "%s %s: stripe offset and count must be specified\n", progname, argv[0]); return CMD_HELP; } - if (stripe_offset_opt != NULL) { - /* get the stripe offset */ - stripe_offset = strtoul(stripe_offset_opt, &end, 0); - if (*end != '\0') { - fprintf(stderr, - "%s %s: bad stripe offset '%s'\n", - progname, argv[0], stripe_offset_opt); - return CMD_HELP; - } - } - - if (delete) { - if (stripe_offset_opt != NULL || stripe_count_opt != NULL) { - fprintf(stderr, - "%s %s: cannot specify -d with -c or -i options\n", - progname, argv[0]); - return CMD_HELP; - } else { - stripe_count = 0; - } + if (delete && + (lsa.lsa_stripe_off != LLAPI_LAYOUT_DEFAULT || + lsa.lsa_stripe_count != LLAPI_LAYOUT_DEFAULT)) { + fprintf(stderr, + "%s %s: cannot specify -d with -c or -i options\n", + progname, argv[0]); + return CMD_HELP; } - if (mode_opt != NULL) { mode = strtoul(mode_opt, &end, 8); if (*end != '\0') { @@ -3731,40 +3739,49 @@ static int lfs_setdirstripe(int argc, char **argv) previous_mode = umask(0); } - if (stripe_hash_opt == NULL) { - hash_type = LMV_HASH_TYPE_FNV_1A_64; - } else { - hash_type = check_hashtype(stripe_hash_opt); - if (hash_type == 0) { - fprintf(stderr, "%s %s: bad stripe hash type '%s'\n", - progname, argv[0], stripe_hash_opt); - return CMD_HELP; - } + /* initialize stripe parameters */ + param = calloc(1, offsetof(typeof(*param), lsp_osts[lsa.lsa_nr_tgts])); + if (param == NULL) { + fprintf(stderr, + "%s %s: cannot allocate memory for parameters: %s\n", + progname, argv[0], strerror(ENOMEM)); + return CMD_HELP; } - /* get the stripe count */ - if (stripe_count_opt != NULL) { - stripe_count = strtoul(stripe_count_opt, &end, 0); - if (*end != '\0') { - fprintf(stderr, - "%s %s: bad stripe count '%s'\n", - progname, argv[0], stripe_count_opt); + if (lsa.lsa_stripe_count != LLAPI_LAYOUT_DEFAULT) + param->lsp_stripe_count = lsa.lsa_stripe_count; + if (lsa.lsa_stripe_off == LLAPI_LAYOUT_DEFAULT) + param->lsp_stripe_offset = -1; + else + param->lsp_stripe_offset = lsa.lsa_stripe_off; + if (lsa.lsa_pattern != LLAPI_LAYOUT_RAID0) + param->lsp_stripe_pattern = lsa.lsa_pattern; + else + param->lsp_stripe_pattern = LMV_HASH_TYPE_FNV_1A_64; + param->lsp_pool = lsa.lsa_pool_name; + param->lsp_is_specific = false; + if (lsa.lsa_nr_tgts > 1) { + if (lsa.lsa_stripe_count > 0 && + lsa.lsa_stripe_count != LLAPI_LAYOUT_DEFAULT && + lsa.lsa_stripe_count != lsa.lsa_nr_tgts) { + fprintf(stderr, "error: %s: stripe count %lld doesn't " + "match the number of MDTs: %d\n", + argv[0], lsa.lsa_stripe_count, lsa.lsa_nr_tgts); + free(param); return CMD_HELP; } + + param->lsp_is_specific = true; + param->lsp_stripe_count = lsa.lsa_nr_tgts; + memcpy(param->lsp_tgts, mdts, sizeof(*mdts) * lsa.lsa_nr_tgts); } dname = argv[optind]; do { - if (default_stripe) { - result = llapi_dir_set_default_lmv_stripe(dname, - stripe_offset, stripe_count, - hash_type, NULL); - } else { - result = llapi_dir_create_pool(dname, mode, - stripe_offset, - stripe_count, hash_type, - NULL); - } + if (default_stripe) + result = llapi_dir_set_default_lmv(dname, param); + else + result = llapi_dir_create_param(dname, mode, param); if (result) { fprintf(stderr, @@ -3778,6 +3795,7 @@ static int lfs_setdirstripe(int argc, char **argv) if (mode_opt != NULL) umask(previous_mode); + free(param); return result; } diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index e0fca12..86b2a83 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -390,6 +390,33 @@ int llapi_stripe_limit_check(unsigned long long stripe_size, int stripe_offset, return 0; } +int llapi_dir_stripe_limit_check(int stripe_offset, int stripe_count, + int hash_type) +{ + int rc; + + if (!llapi_dir_stripe_index_is_valid(stripe_offset)) { + rc = -EINVAL; + llapi_error(LLAPI_MSG_ERROR, rc, "error: bad stripe offset %d", + stripe_offset); + return rc; + } + if (!llapi_dir_stripe_count_is_valid(stripe_count)) { + rc = -EINVAL; + llapi_error(LLAPI_MSG_ERROR, rc, "error: bad stripe count %d", + stripe_count); + return rc; + } + + if (!llapi_dir_hash_type_is_valid(hash_type)) { + rc = -EINVAL; + llapi_error(LLAPI_MSG_ERROR, rc, "error: bad hash type %d", + hash_type); + return rc; + } + return 0; +} + /* * Trim a trailing newline from a string, if it exists. */ @@ -469,13 +496,13 @@ int llapi_get_agent_uuid(char *path, char *buf, size_t bufsize) } /* - * if pool is NULL, search ostname in target_obd + * if pool is NULL, search tgtname in target_obd * if pool is not NULL: * if pool not found returns errno < 0 - * if ostname is NULL, returns 1 if pool is not empty and 0 if pool empty - * if ostname is not NULL, returns 1 if OST is in pool and 0 if not + * if tgtname is NULL, returns 1 if pool is not empty and 0 if pool empty + * if tgtname is not NULL, returns 1 if OST is in pool and 0 if not */ -int llapi_search_ost(char *fsname, char *poolname, char *ostname) +int llapi_search_tgt(char *fsname, char *poolname, char *tgtname, bool is_mdt) { char buffer[PATH_MAX]; size_t len = 0; @@ -487,8 +514,8 @@ int llapi_search_ost(char *fsname, char *poolname, char *ostname) if (poolname == NULL && fsname == NULL) return -EINVAL; - if (ostname != NULL) - len = strlen(ostname); + if (tgtname != NULL) + len = strlen(tgtname); if (poolname == NULL && len == 0) return -EINVAL; @@ -501,7 +528,7 @@ int llapi_search_ost(char *fsname, char *poolname, char *ostname) param.gl_pathv[0], poolname); } } else if (fsname != NULL) { - rc = get_lustre_param_path("lov", fsname, + rc = get_lustre_param_path(is_mdt ? "lmv" : "lov", fsname, FILTER_BY_FS_NAME, "target_obd", ¶m); if (rc == 0) { @@ -515,33 +542,38 @@ int llapi_search_ost(char *fsname, char *poolname, char *ostname) if (rc) return rc; - fd = fopen(buffer, "r"); - if (fd == NULL) - return -errno; + fd = fopen(buffer, "r"); + if (fd == NULL) + return -errno; - while (fgets(buffer, sizeof(buffer), fd) != NULL) { - if (poolname == NULL) { - char *ptr; - /* Search for an ostname in the list of OSTs - Line format is IDX: fsname-OSTxxxx_UUID STATUS */ - ptr = strchr(buffer, ' '); - if ((ptr != NULL) && - (strncmp(ptr + 1, ostname, len) == 0)) { - fclose(fd); - return 1; - } - } else { - /* Search for an ostname in a pool, - (or an existing non-empty pool if no ostname) */ - if ((ostname == NULL) || - (strncmp(buffer, ostname, len) == 0)) { - fclose(fd); - return 1; - } - } - } - fclose(fd); - return 0; + while (fgets(buffer, sizeof(buffer), fd) != NULL) { + if (poolname == NULL) { + char *ptr; + /* Search for an tgtname in the list of targets + * Line format is IDX: fsname-OST/MDTxxxx_UUID STATUS */ + ptr = strchr(buffer, ' '); + if ((ptr != NULL) && + (strncmp(ptr + 1, tgtname, len) == 0)) { + fclose(fd); + return 1; + } + } else { + /* Search for an tgtname in a pool, + * (or an existing non-empty pool if no tgtname) */ + if ((tgtname == NULL) || + (strncmp(buffer, tgtname, len) == 0)) { + fclose(fd); + return 1; + } + } + } + fclose(fd); + return 0; +} + +int llapi_search_ost(char *fsname, char *poolname, char *ostname) +{ + return llapi_search_tgt(fsname, poolname, ostname, false); } /** @@ -772,29 +804,131 @@ int llapi_file_create_pool(const char *name, unsigned long long stripe_size, return 0; } -int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, - int stripe_count, int stripe_pattern, - const char *pool_name) +static int verify_dir_param(const char *name, + const struct llapi_stripe_param *param) { - struct lmv_user_md lum = { 0 }; - int fd; - int rc = 0; + char fsname[MAX_OBD_NAME + 1] = { 0 }; + char *pool_name = param->lsp_pool; + int rc; + + /* Make sure we are on a Lustre file system */ + rc = llapi_search_fsname(name, fsname); + if (rc) { + llapi_error(LLAPI_MSG_ERROR, rc, + "'%s' is not on a Lustre filesystem", + name); + return rc; + } + + /* Check if the stripe pattern is sane. */ + rc = llapi_dir_stripe_limit_check(param->lsp_stripe_offset, + param->lsp_stripe_count, + param->lsp_stripe_pattern); + if (rc != 0) + return rc; - lum.lum_magic = LMV_USER_MAGIC; - lum.lum_stripe_offset = stripe_offset; - lum.lum_stripe_count = stripe_count; - lum.lum_hash_type = stripe_pattern; + /* Make sure we have a good pool */ if (pool_name != NULL) { - if (strlen(pool_name) >= sizeof(lum.lum_pool_name)) { - llapi_err_noerrno(LLAPI_MSG_ERROR, - "error LL_IOC_LMV_SET_DEFAULT_STRIPE '%s'" - ": too large pool name: %s", name, pool_name); - return -E2BIG; + /* in case user gives the full pool name ., + * strip the fsname */ + char *ptr = strchr(pool_name, '.'); + + if (ptr != NULL) { + *ptr = '\0'; + if (strcmp(pool_name, fsname) != 0) { + *ptr = '.'; + llapi_err_noerrno(LLAPI_MSG_ERROR, + "Pool '%s' is not on filesystem '%s'", + pool_name, fsname); + return -EINVAL; + } + pool_name = ptr + 1; + } + + /* Make sure the pool exists and is non-empty */ + rc = llapi_search_tgt(fsname, pool_name, NULL, true); + if (rc < 1) { + char *err = rc == 0 ? "has no OSTs" : "does not exist"; + + llapi_err_noerrno(LLAPI_MSG_ERROR, "pool '%s.%s' %s", + fsname, pool_name, err); + return -EINVAL; } - strncpy(lum.lum_pool_name, pool_name, - sizeof(lum.lum_pool_name)); } + /* sanity check of target list */ + if (param->lsp_is_specific) { + char mdtname[MAX_OBD_NAME + 1]; + bool found = false; + int i; + + for (i = 0; i < param->lsp_stripe_count; i++) { + snprintf(mdtname, sizeof(mdtname), "%s-MDT%04x_UUID", + fsname, param->lsp_tgts[i]); + rc = llapi_search_tgt(fsname, pool_name, mdtname, true); + if (rc <= 0) { + if (rc == 0) + rc = -ENODEV; + + llapi_error(LLAPI_MSG_ERROR, rc, + "%s: cannot find MDT %s in %s", + __func__, mdtname, + pool_name != NULL ? + "pool" : "system"); + return rc; + } + + /* Make sure stripe offset is in MDT list. */ + if (param->lsp_tgts[i] == param->lsp_stripe_offset) + found = true; + } + if (!found) { + llapi_error(LLAPI_MSG_ERROR, -EINVAL, + "%s: stripe offset '%d' is not in the " + "target list", + __func__, param->lsp_stripe_offset); + return -EINVAL; + } + } + + return 0; +} + +static inline void param2lmu(struct lmv_user_md *lmu, + const struct llapi_stripe_param *param) +{ + lmu->lum_magic = param->lsp_is_specific ? LMV_USER_MAGIC_SPECIFIC : + LMV_USER_MAGIC; + lmu->lum_stripe_count = param->lsp_stripe_count; + lmu->lum_stripe_offset = param->lsp_stripe_offset; + lmu->lum_hash_type = param->lsp_stripe_pattern; + if (param->lsp_pool != NULL) + strncpy(lmu->lum_pool_name, param->lsp_pool, LOV_MAXPOOLNAME); + if (param->lsp_is_specific) { + int i; + + for (i = 0; i < param->lsp_stripe_count; i++) + lmu->lum_objects[i].lum_mds = param->lsp_tgts[i]; + } +} + +int llapi_dir_set_default_lmv(const char *name, + const struct llapi_stripe_param *param) +{ + struct lmv_user_md lmu = { 0 }; + int fd; + int rc = 0; + + rc = verify_dir_param(name, param); + if (rc) + return rc; + + /* TODO: default lmv doesn't support specific targets yet */ + if (param->lsp_is_specific) + return -EINVAL; + + param2lmu(&lmu, param); + fd = open(name, O_DIRECTORY | O_RDONLY); if (fd < 0) { rc = -errno; @@ -802,7 +936,7 @@ int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, return rc; } - rc = ioctl(fd, LL_IOC_LMV_SET_DEFAULT_STRIPE, &lum); + rc = ioctl(fd, LL_IOC_LMV_SET_DEFAULT_STRIPE, &lmu); if (rc < 0) { char *errmsg = "stripe already set"; rc = -errno; @@ -817,11 +951,35 @@ int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, return rc; } -int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, - int stripe_count, int stripe_pattern, - const char *pool_name) +int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, + int stripe_count, int stripe_pattern, + const char *pool_name) { - struct lmv_user_md lmu = { 0 }; + const struct llapi_stripe_param param = { + .lsp_stripe_count = stripe_count, + .lsp_stripe_offset = stripe_offset, + .lsp_stripe_pattern = stripe_pattern, + .lsp_pool = (char *)pool_name + }; + + return llapi_dir_set_default_lmv(name, ¶m); +} + +/** + * Create a Lustre directory. + * + * \param name the name of the directory to be created + * \param mode permission of the file if it is created, see mode in open(2) + * \param param stripe pattern of the newly created directory + * + * \retval 0 on success + * \retval negative errno on failure + */ +int llapi_dir_create_param(const char *name, mode_t mode, + const struct llapi_stripe_param *param) +{ + struct lmv_user_md *lmu = NULL; + size_t lmu_size = sizeof(*lmu); struct obd_ioctl_data data = { 0 }; char rawbuf[8192]; char *buf = rawbuf; @@ -829,36 +987,42 @@ int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, char *namepath = NULL; char *dir; char *filename; - int fd = -1; - int rc; + int fd, rc; + + rc = verify_dir_param(name, param); + if (rc) + return rc; + + if (param->lsp_is_specific) + lmu_size = lmv_user_md_size(param->lsp_stripe_count, + LMV_USER_MAGIC_SPECIFIC); + + lmu = calloc(1, lmu_size); + if (lmu == NULL) + return -ENOMEM; dirpath = strdup(name); - namepath = strdup(name); - if (!dirpath || !namepath) + if (!dirpath) { + free(lmu); return -ENOMEM; + } - lmu.lum_magic = LMV_USER_MAGIC; - lmu.lum_stripe_offset = stripe_offset; - lmu.lum_stripe_count = stripe_count; - lmu.lum_hash_type = stripe_pattern; - if (pool_name != NULL) { - if (strlen(pool_name) > LOV_MAXPOOLNAME) { - llapi_err_noerrno(LLAPI_MSG_ERROR, - "error LL_IOC_LMV_SETSTRIPE '%s' : too large" - "pool name: %s", name, pool_name); - rc = -E2BIG; - goto out; - } - memcpy(lmu.lum_pool_name, pool_name, strlen(pool_name)); + namepath = strdup(name); + if (!namepath) { + free(namepath); + free(lmu); + return -ENOMEM; } + param2lmu(lmu, param); + filename = basename(namepath); dir = dirname(dirpath); data.ioc_inlbuf1 = (char *)filename; data.ioc_inllen1 = strlen(filename) + 1; - data.ioc_inlbuf2 = (char *)&lmu; - data.ioc_inllen2 = sizeof(struct lmv_user_md); + data.ioc_inlbuf2 = (char *)lmu; + data.ioc_inllen2 = lmu_size; data.ioc_type = mode; rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf)); if (rc) { @@ -887,11 +1051,27 @@ int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, } close(fd); out: - free(dirpath); free(namepath); + free(dirpath); + free(lmu); return rc; } + +int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, + int stripe_count, int stripe_pattern, + const char *pool_name) +{ + const struct llapi_stripe_param param = { + .lsp_stripe_count = stripe_count, + .lsp_stripe_offset = stripe_offset, + .lsp_stripe_pattern = stripe_pattern, + .lsp_pool = (char *)pool_name + }; + + return llapi_dir_create_param(name, mode, ¶m); +} + int llapi_direntry_remove(char *dname) { char *dirpath = NULL; diff --git a/lustre/utils/lustreapi_internal.h b/lustre/utils/lustreapi_internal.h index 3b94298..170f125 100644 --- a/lustre/utils/lustreapi_internal.h +++ b/lustre/utils/lustreapi_internal.h @@ -134,6 +134,21 @@ static inline bool llapi_stripe_index_is_valid(int64_t index) * terminology instead of the preferred "index". */ #define llapi_stripe_offset_is_valid(os) llapi_stripe_index_is_valid(os) +static inline bool llapi_dir_stripe_count_is_valid(int64_t count) +{ + return count >= -1 && count <= LMV_MAX_STRIPE_COUNT; +} + +static inline bool llapi_dir_stripe_index_is_valid(int64_t index) +{ + return index >= -1 && index < LMV_MAX_STRIPE_COUNT; +} + +static inline bool llapi_dir_hash_type_is_valid(int64_t hash) +{ + return hash > LMV_HASH_TYPE_UNKNOWN && hash < LMV_HASH_TYPE_MAX; +} + /* * Kernel communication for Changelogs and HSM requests. */ -- 1.8.3.1