From 01d34a6b3b2e34f7414f627e4f87993322dafa78 Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Mon, 15 Mar 2021 11:57:36 +0800 Subject: [PATCH] LU-13440 lmv: add default LMV inherit depth A new field "__u8 lum_max_inherit" is added into struct lmv_user_md, which represents the inherit depth of default LMV. It will be decreased by 1 for subdirectories. The valid value of lum_max_inherit is [0, 255]: * 0 means unlimited inherit. * 1 means inherit end. * 250 is the max inherit depth. * [251, 254] are reserved. * 255 means it's not set. A new field "__u8 lum_max_inherit_rr" is added, if default stripe offset is -1, lum_max_inherit_rr is non-zero, and system is balanced, new directories are created in roundrobin mannner, otherwise they are created on the MDT where their parents are located to avoid creating remote directories. And similarly this value will be decreased by 1 for each level of subdirectories. The valid value of lum_max_inherit_rr is different: * 0 means not set. * 1 means inherit end. * 250 is the max inherit depth. * [251, 254] are reserved. * 255 means unlimited inherit. However for the user interface of "lfs", the valid value is [-1, 250]: * -1 means unlimited inherit. * 0 means not set. * others are the same. Add sanity 413c. Signed-off-by: Lai Siyao Change-Id: I98ccad8556a0469f83bd7d79f5086a2184d5b115 Reviewed-on: https://review.whamcloud.com/43131 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Hongchao Zhang --- lustre/doc/lfs-getdirstripe.1 | 8 +- lustre/doc/lfs-setdirstripe.1 | 36 ++++++- lustre/include/lu_object.h | 24 ++++- lustre/include/lustre/lustreapi.h | 45 +++++---- lustre/include/lustre_lmv.h | 8 +- lustre/include/uapi/linux/lustre/lustre_user.h | 39 +++++++- lustre/llite/namei.c | 4 + lustre/lmv/lmv_obd.c | 62 ++++++++++-- lustre/lod/lod_internal.h | 23 +++++ lustre/lod/lod_object.c | 8 ++ lustre/obdclass/lu_tgt_descs.c | 16 ---- lustre/ptlrpc/pack_generic.c | 3 + lustre/tests/sanity.sh | 114 ++++++++++++++++------ lustre/utils/lfs.c | 126 ++++++++++++++++++++++--- lustre/utils/liblustreapi.c | 45 ++++++++- 15 files changed, 461 insertions(+), 100 deletions(-) diff --git a/lustre/doc/lfs-getdirstripe.1 b/lustre/doc/lfs-getdirstripe.1 index 7727020..0636e41 100644 --- a/lustre/doc/lfs-getdirstripe.1 +++ b/lustre/doc/lfs-getdirstripe.1 @@ -2,7 +2,7 @@ .SH NAME lfs getdirstripe \- list the layout pattern of a given directory .SH SYNOPSIS -.BR "lfs getdirstripe" " [" -cDimOrTy "]... " \fIDIR\fR... +.BR "lfs getdirstripe" " [" -cDimOrTXy "]... " \fIDIR\fR... .SH DESCRIPTION Get the layout pattern of striped directories. This .BR lfs (1) @@ -35,8 +35,14 @@ can be expanded with .BR --recursive , which will recurse into all subdirectories. .TP +.BR \-X ", " \-\-max-inherit +Show the inherit depth of default layout. +.TP .BR \-y ", " \-\-yaml Show the layout in YAML format for easier parsing. +.TP +.BR \-\-max-inherit-rr +Show the inherit depth of default layout to roundrobin mkdir. .SH EXAMPLES .TP .B $ lfs getdirstripe -O lustre-MDT0001_UUID -r /mnt/lustre/dir1 diff --git a/lustre/doc/lfs-setdirstripe.1 b/lustre/doc/lfs-setdirstripe.1 index d61240e..81d413f 100644 --- a/lustre/doc/lfs-setdirstripe.1 +++ b/lustre/doc/lfs-setdirstripe.1 @@ -2,7 +2,7 @@ .SH NAME lfs setdirstripe, mkdir \- set striping pattern of a directory. .SH SYNOPSIS -.B lfs setdirstripe [\fR-cdDHioTx\fR] \fIDIR\fR... +.B lfs setdirstripe [\fR-cdDHioTxX\fR] \fIDIR\fR... .br .SH DESCRIPTION Create a striped directory with specified striping pattern. This @@ -104,6 +104,40 @@ Specify a numeric bitmask of type-specific layout flags for the foreign layout. .TP .BR \-x ", " \-\-xattr =\fISTRING\fR Specify a string to be used as a foreign (free format) striping. +.TP +.BR \-X ", " \-\-max-inherit = \fIMAX_INHERIT +Set the inherit depth of default directory layout. If non-zero, then +.I MAX_INHERIT +is the number of subdirectory levels for which this default layout is inherited, +up to a maximum of 250 levels, and is decremented by one when copying the +default layout to each new subdirectory, until zero and the default layout is +no longer copied. A +.I MAX_INHERIT +of -1 means the default layout is inherited for all subdirectories. +.TP +.BR \-\-max-inherit-rr = \fIMAX_INHERIT_RR +Set the round-robin inherit depth of the default directory layout, only when +.I START_MDT_INDEX +is -1. If +.I MAX_INHERIT_RR +is zero (unset), then new subdirectories are preferentially created on MDTs +with more free space and inodes if the MDTs are imbalanced. If non-zero, then +.I MAX_INHERIT_RR +is the number of subdirectory levels, up to a maximum of 250 levels, for which +new subdirectories will be created in a round-robin manner across all available +MDTs, rather than using MDT space balancing for new subdirectories. Otherwise, +.I MAX_INHERIT_RR +is decremented by one when copying the default layout to each new subdirectory, +until zero and round-robin is no longer used. It may be useful to set +.I MAX_INHERIT_RR +to 2 or 3 when setting the default directory layout on the root of a +.B new +filesystem, so that top-level subdirectories are immediately spread +across MDTs, rather than waiting for the MDTs to become imbalanced. +Round-robin subdirectory creation is unlikely to be useful for existing +filesystems that already have many files and imbalanced MDTs, since space +balancing will already happen, and this will unnecessarily increase the number +of remote subdirectories (increasing overhead) without any benefit. .SH NOTE .PP If neither diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 01a3e0d..884d79e 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -1677,11 +1677,33 @@ int lu_tgt_descs_init(struct lu_tgt_descs *ltd, bool is_mdt); void lu_tgt_descs_fini(struct lu_tgt_descs *ltd); int ltd_add_tgt(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt); void ltd_del_tgt(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt); -bool ltd_qos_is_usable(struct lu_tgt_descs *ltd); int ltd_qos_penalties_calc(struct lu_tgt_descs *ltd); int ltd_qos_update(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt, __u64 *total_wt); +/** + * Whether MDT inode and space usages are balanced. + */ +static inline bool ltd_qos_is_balanced(struct lu_tgt_descs *ltd) +{ + return !test_bit(LQ_DIRTY, <d->ltd_qos.lq_flags) && + test_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags); +} + +/** + * Whether QoS data is up-to-date and QoS can be applied. + */ +static inline bool ltd_qos_is_usable(struct lu_tgt_descs *ltd) +{ + if (ltd_qos_is_balanced(ltd)) + return false; + + if (ltd->ltd_lov_desc.ld_active_tgt_count < 2) + return false; + + return true; +} + static inline struct lu_tgt_desc *ltd_first_tgt(struct lu_tgt_descs *ltd) { int index; diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 8beeefc..4d7c299 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -141,6 +141,8 @@ struct llapi_stripe_param { int lsp_stripe_count; bool lsp_is_specific; bool lsp_is_create; + __u8 lsp_max_inherit; + __u8 lsp_max_inherit_rr; __u32 lsp_osts[0]; }; @@ -174,25 +176,27 @@ void llapi_set_command_name(const char *cmd); void llapi_clear_command_name(void); enum llapi_layout_verbose { - VERBOSE_STRIPE_COUNT = 0x1, - VERBOSE_STRIPE_SIZE = 0x2, - VERBOSE_STRIPE_OFFSET = 0x4, - VERBOSE_POOL = 0x8, - VERBOSE_DETAIL = 0x10, - VERBOSE_OBJID = 0x20, - VERBOSE_GENERATION = 0x40, - VERBOSE_MDTINDEX = 0x80, - VERBOSE_PATTERN = 0x100, - VERBOSE_COMP_COUNT = 0x200, - VERBOSE_COMP_FLAGS = 0x400, - VERBOSE_COMP_START = 0x800, - VERBOSE_COMP_END = 0x1000, - VERBOSE_COMP_ID = 0x2000, - VERBOSE_DFID = 0x4000, - VERBOSE_HASH_TYPE = 0x8000, - VERBOSE_MIRROR_COUNT = 0x10000, - VERBOSE_MIRROR_ID = 0x20000, - VERBOSE_EXT_SIZE = 0x40000, + VERBOSE_STRIPE_COUNT = 0x1, + VERBOSE_STRIPE_SIZE = 0x2, + VERBOSE_STRIPE_OFFSET = 0x4, + VERBOSE_POOL = 0x8, + VERBOSE_DETAIL = 0x10, + VERBOSE_OBJID = 0x20, + VERBOSE_GENERATION = 0x40, + VERBOSE_MDTINDEX = 0x80, + VERBOSE_PATTERN = 0x100, + VERBOSE_COMP_COUNT = 0x200, + VERBOSE_COMP_FLAGS = 0x400, + VERBOSE_COMP_START = 0x800, + VERBOSE_COMP_END = 0x1000, + VERBOSE_COMP_ID = 0x2000, + VERBOSE_DFID = 0x4000, + VERBOSE_HASH_TYPE = 0x8000, + VERBOSE_MIRROR_COUNT = 0x10000, + VERBOSE_MIRROR_ID = 0x20000, + VERBOSE_EXT_SIZE = 0x40000, + VERBOSE_INHERIT = 0x80000, + VERBOSE_INHERIT_RR = 0x100000, VERBOSE_DEFAULT = VERBOSE_STRIPE_COUNT | VERBOSE_STRIPE_SIZE | VERBOSE_STRIPE_OFFSET | VERBOSE_POOL | VERBOSE_OBJID | VERBOSE_GENERATION | @@ -200,7 +204,8 @@ enum llapi_layout_verbose { VERBOSE_COMP_COUNT | VERBOSE_COMP_FLAGS | VERBOSE_COMP_START | VERBOSE_COMP_END | VERBOSE_COMP_ID | VERBOSE_MIRROR_COUNT | - VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE + VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE | + VERBOSE_INHERIT | VERBOSE_INHERIT_RR }; /* Compatibility with original names */ #define VERBOSE_SIZE VERBOSE_STRIPE_SIZE diff --git a/lustre/include/lustre_lmv.h b/lustre/include/lustre_lmv.h index a3f8ed4..811da36 100644 --- a/lustre/include/lustre_lmv.h +++ b/lustre/include/lustre_lmv.h @@ -45,6 +45,8 @@ struct lmv_stripe_md { __u32 lsm_md_stripe_count; __u32 lsm_md_master_mdt_index; __u32 lsm_md_hash_type; + __u8 lsm_md_max_inherit; + __u8 lsm_md_max_inherit_rr; __u32 lsm_md_layout_version; __u32 lsm_md_migrate_offset; __u32 lsm_md_migrate_hash; @@ -120,11 +122,11 @@ static inline void lsm_md_dump(int mask, const struct lmv_stripe_md *lsm) * terminated string so only print LOV_MAXPOOLNAME bytes. */ CDEBUG(mask, - "magic %#x stripe count %d master mdt %d hash type %#x version %d migrate offset %d migrate hash %#x pool %.*s\n", + "magic %#x stripe count %d master mdt %d hash type %#x max inherit %hhu version %d migrate offset %d migrate hash %#x pool %.*s\n", lsm->lsm_md_magic, lsm->lsm_md_stripe_count, lsm->lsm_md_master_mdt_index, lsm->lsm_md_hash_type, - lsm->lsm_md_layout_version, lsm->lsm_md_migrate_offset, - lsm->lsm_md_migrate_hash, + lsm->lsm_md_max_inherit, lsm->lsm_md_layout_version, + lsm->lsm_md_migrate_offset, lsm->lsm_md_migrate_hash, LOV_MAXPOOLNAME, lsm->lsm_md_pool_name); if (!lmv_dir_striped(lsm)) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 24867af..9297f0d 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -1086,12 +1086,14 @@ extern struct lustre_foreign_type lu_foreign_types[]; #define LMV_MAX_STRIPE_COUNT 2000 /* ((12 * 4096 - 256) / 24) */ #define lmv_user_md lmv_user_md_v1 struct lmv_user_md_v1 { - __u32 lum_magic; /* must be the first field */ + __u32 lum_magic; /* must be the first field */ __u32 lum_stripe_count; /* dirstripe count */ __u32 lum_stripe_offset; /* MDT idx for default dirstripe */ __u32 lum_hash_type; /* Dir stripe policy */ - __u32 lum_type; /* LMV type: default */ - __u32 lum_padding1; + __u32 lum_type; /* LMV type: default */ + __u8 lum_max_inherit; /* inherit depth of default LMV */ + __u8 lum_max_inherit_rr; /* inherit depth of default LMV to round-robin mkdir */ + __u16 lum_padding1; __u32 lum_padding2; __u32 lum_padding3; char lum_pool_name[LOV_MAXPOOLNAME + 1]; @@ -1117,6 +1119,37 @@ enum lmv_type { LMV_TYPE_DEFAULT = 0x0000, }; +/* lum_max_inherit will be decreased by 1 after each inheritance if it's not + * LMV_INHERIT_UNLIMITED or > LMV_INHERIT_MAX. + */ +enum { + /* for historical reason, 0 means unlimited inheritance */ + LMV_INHERIT_UNLIMITED = 0, + /* unlimited lum_max_inherit by default */ + LMV_INHERIT_DEFAULT = 0, + /* not inherit any more */ + LMV_INHERIT_END = 1, + /* max inherit depth */ + LMV_INHERIT_MAX = 250, + /* [251, 254] are reserved */ + /* not set, or when inherit depth goes beyond end, */ + LMV_INHERIT_NONE = 255, +}; + +enum { + /* not set, or when inherit_rr depth goes beyond end, */ + LMV_INHERIT_RR_NONE = 0, + /* disable lum_max_inherit_rr by default */ + LMV_INHERIT_RR_DEFAULT = 0, + /* not inherit any more */ + LMV_INHERIT_RR_END = 1, + /* max inherit depth */ + LMV_INHERIT_RR_MAX = 250, + /* [251, 254] are reserved */ + /* unlimited inheritance */ + LMV_INHERIT_RR_UNLIMITED = 255, +}; + static inline int lmv_user_md_size(int stripes, int lmm_magic) { int size = sizeof(struct lmv_user_md); diff --git a/lustre/llite/namei.c b/lustre/llite/namei.c index e34d1ee..4889e89 100644 --- a/lustre/llite/namei.c +++ b/lustre/llite/namei.c @@ -1460,6 +1460,10 @@ again: md.default_lmv->lsm_md_master_mdt_index = lum->lum_stripe_offset; md.default_lmv->lsm_md_hash_type = lum->lum_hash_type; + md.default_lmv->lsm_md_max_inherit = + lum->lum_max_inherit; + md.default_lmv->lsm_md_max_inherit_rr = + lum->lum_max_inherit_rr; err = ll_update_inode(dir, &md); md_free_lustre_md(sbi->ll_md_exp, &md); diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 788e88c..5236e44 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -1720,6 +1720,22 @@ int lmv_old_layout_lookup(struct lmv_obd *lmv, struct md_op_data *op_data) return rc; } +static inline bool lmv_op_user_qos_mkdir(const struct md_op_data *op_data) +{ + const struct lmv_user_md *lum = op_data->op_data; + + return (op_data->op_cli_flags & CLI_SET_MEA) && lum && + le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC && + le32_to_cpu(lum->lum_stripe_offset) == LMV_OFFSET_DEFAULT; +} + +static inline bool lmv_op_default_qos_mkdir(const struct md_op_data *op_data) +{ + const struct lmv_stripe_md *lsm = op_data->op_default_mea1; + + return lsm && lsm->lsm_md_master_mdt_index == LMV_OFFSET_DEFAULT; +} + /* mkdir by QoS in two cases: * 1. 'lfs mkdir -i -1' * 2. parent default LMV master_mdt_index is -1 @@ -1729,27 +1745,38 @@ int lmv_old_layout_lookup(struct lmv_obd *lmv, struct md_op_data *op_data) */ static inline bool lmv_op_qos_mkdir(const struct md_op_data *op_data) { - const struct lmv_stripe_md *lsm = op_data->op_default_mea1; - const struct lmv_user_md *lum = op_data->op_data; - if (op_data->op_code != LUSTRE_OPC_MKDIR) return false; if (lmv_dir_striped(op_data->op_mea1)) return false; - if (op_data->op_cli_flags & CLI_SET_MEA && lum && - (le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC || - le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC_SPECIFIC) && - le32_to_cpu(lum->lum_stripe_offset) == LMV_OFFSET_DEFAULT) + if (lmv_op_user_qos_mkdir(op_data)) return true; - if (lsm && lsm->lsm_md_master_mdt_index == LMV_OFFSET_DEFAULT) + if (lmv_op_default_qos_mkdir(op_data)) return true; return false; } +/* if default LMV is set, and its index is LMV_OFFSET_DEFAULT, and + * 1. max_inherit_rr is set and is not LMV_INHERIT_RR_NONE + * 2. or parent is ROOT + * mkdir roundrobin. + * NB, this also needs to check server is balanced, which is checked by caller. + */ +static inline bool lmv_op_default_rr_mkdir(const struct md_op_data *op_data) +{ + const struct lmv_stripe_md *lsm = op_data->op_default_mea1; + + if (!lmv_op_default_qos_mkdir(op_data)) + return false; + + return lsm->lsm_md_max_inherit_rr != LMV_INHERIT_RR_NONE || + fid_is_root(&op_data->op_fid1); +} + /* 'lfs mkdir -i ' */ static inline bool lmv_op_user_specific_mkdir(const struct md_op_data *op_data) { @@ -1771,6 +1798,7 @@ lmv_op_default_specific_mkdir(const struct md_op_data *op_data) op_data->op_default_mea1->lsm_md_master_mdt_index != LMV_OFFSET_DEFAULT; } + int lmv_create(struct obd_export *exp, struct md_op_data *op_data, const void *data, size_t datalen, umode_t mode, uid_t uid, gid_t gid, cfs_cap_t cap_effective, __u64 rdev, @@ -1820,11 +1848,23 @@ int lmv_create(struct obd_export *exp, struct md_op_data *op_data, if (!tgt) RETURN(-ENODEV); } else if (lmv_op_qos_mkdir(op_data)) { + struct lmv_tgt_desc *tmp = tgt; + tgt = lmv_locate_tgt_qos(lmv, &op_data->op_mds); - if (tgt == ERR_PTR(-EAGAIN)) - tgt = lmv_locate_tgt_rr(lmv, &op_data->op_mds); + if (tgt == ERR_PTR(-EAGAIN)) { + if (ltd_qos_is_balanced(&lmv->lmv_mdt_descs) && + !lmv_op_default_rr_mkdir(op_data) && + !lmv_op_user_qos_mkdir(op_data)) + /* if it's not necessary, don't create remote + * directory. + */ + tgt = tmp; + else + tgt = lmv_locate_tgt_rr(lmv, &op_data->op_mds); + } if (IS_ERR(tgt)) RETURN(PTR_ERR(tgt)); + /* * only update statfs after QoS mkdir, this means the cached * statfs may be stale, and current mkdir may not follow QoS @@ -3161,6 +3201,8 @@ static inline int lmv_unpack_user_md(struct obd_export *exp, lsm->lsm_md_stripe_count = le32_to_cpu(lmu->lum_stripe_count); lsm->lsm_md_master_mdt_index = le32_to_cpu(lmu->lum_stripe_offset); lsm->lsm_md_hash_type = le32_to_cpu(lmu->lum_hash_type); + lsm->lsm_md_max_inherit = lmu->lum_max_inherit; + lsm->lsm_md_max_inherit_rr = lmu->lum_max_inherit_rr; lsm->lsm_md_pool_name[LOV_MAXPOOLNAME] = 0; return 0; diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 241a77c..53664b9 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -182,6 +182,8 @@ struct lod_default_striping { __u32 lds_dir_def_stripe_count; __u32 lds_dir_def_stripe_offset; __u32 lds_dir_def_hash_type; + __u8 lds_dir_def_max_inherit; + __u8 lds_dir_def_max_inherit_rr; /* default file striping flags (LOV) */ __u32 lds_def_striping_set:1, lds_def_striping_is_composite:1, @@ -189,6 +191,27 @@ struct lod_default_striping { lds_dir_def_striping_set:1; }; +static inline __u8 lmv_inherit_next(__u8 inherit) +{ + if (inherit == LMV_INHERIT_END || inherit == LMV_INHERIT_NONE) + return LMV_INHERIT_NONE; + + if (inherit == LMV_INHERIT_UNLIMITED || inherit > LMV_INHERIT_MAX) + return inherit; + + return inherit - 1; +} + +static inline __u8 lmv_inherit_rr_next(__u8 inherit_rr) +{ + if (inherit_rr == LMV_INHERIT_RR_NONE || + inherit_rr == LMV_INHERIT_RR_UNLIMITED || + inherit_rr > LMV_INHERIT_RR_MAX) + return inherit_rr; + + return inherit_rr - 1; +} + struct lod_mirror_entry { __u16 lme_stale:1, lme_prefer:1; diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index f4829a3..38d04ff 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -4252,6 +4252,8 @@ static int lod_dir_striping_create_internal(const struct lu_env *env, /* Transfer default LMV striping from the parent */ if (lds != NULL && lds->lds_dir_def_striping_set && + lds->lds_dir_def_max_inherit != LMV_INHERIT_END && + lds->lds_dir_def_max_inherit != LMV_INHERIT_NONE && !(LMVEA_DELETE_VALUES(lds->lds_dir_def_stripe_count, lds->lds_dir_def_stripe_offset) && le32_to_cpu(lds->lds_dir_def_hash_type) != @@ -4273,6 +4275,10 @@ static int lod_dir_striping_create_internal(const struct lu_env *env, cpu_to_le32(lds->lds_dir_def_stripe_offset); v1->lum_hash_type = cpu_to_le32(lds->lds_dir_def_hash_type); + v1->lum_max_inherit = + lmv_inherit_next(lds->lds_dir_def_max_inherit); + v1->lum_max_inherit_rr = + lmv_inherit_rr_next(lds->lds_dir_def_max_inherit_rr); info->lti_buf.lb_buf = v1; info->lti_buf.lb_len = sizeof(*v1); @@ -5195,6 +5201,8 @@ static int lod_get_default_lmv_striping(const struct lu_env *env, le32_to_cpu(lmu->lum_stripe_offset); lds->lds_dir_def_hash_type = le32_to_cpu(lmu->lum_hash_type); + lds->lds_dir_def_max_inherit = lmu->lum_max_inherit; + lds->lds_dir_def_max_inherit_rr = lmu->lum_max_inherit_rr; lds->lds_dir_def_striping_set = 1; } diff --git a/lustre/obdclass/lu_tgt_descs.c b/lustre/obdclass/lu_tgt_descs.c index bdffc78..04e2330 100644 --- a/lustre/obdclass/lu_tgt_descs.c +++ b/lustre/obdclass/lu_tgt_descs.c @@ -414,22 +414,6 @@ void ltd_del_tgt(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt) EXPORT_SYMBOL(ltd_del_tgt); /** - * Whether QoS data is up-to-date and QoS can be applied. - */ -bool ltd_qos_is_usable(struct lu_tgt_descs *ltd) -{ - if (!test_bit(LQ_DIRTY, <d->ltd_qos.lq_flags) && - test_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags)) - return false; - - if (ltd->ltd_lov_desc.ld_active_tgt_count < 2) - return false; - - return true; -} -EXPORT_SYMBOL(ltd_qos_is_usable); - -/** * Calculate penalties per-tgt and per-server * * Re-calculate penalties when the configuration changes, active targets diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 74f93fe..bfa1d8f 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2279,7 +2279,10 @@ void lustre_swab_lmv_user_md(struct lmv_user_md *lum) __swab32s(&lum->lum_stripe_offset); __swab32s(&lum->lum_hash_type); __swab32s(&lum->lum_type); + /* lum_max_inherit and lum_max_inherit_rr do not need to be swabbed */ BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding1) == 0); + BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding2) == 0); + BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding3) == 0); switch (lum->lum_magic) { case LMV_USER_MAGIC_SPECIFIC: count = lum->lum_stripe_count; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index cd00e69..a13262b 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -23755,14 +23755,26 @@ test_qos_mkdir() { stack_trap "do_nodes $mdts $LCTL set_param \ lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" EXIT - echo - echo "Mkdir (stripe_count $stripe_count) roundrobin:" - $LCTL set_param lmv.*.qos_threshold_rr=100 > /dev/null do_nodes $mdts $LCTL set_param lod.*.mdt_qos_threshold_rr=100 > /dev/null testdir=$DIR/$tdir-s$stripe_count/rr + local stripe_index=$($LFS getstripe -m $testdir) + local test_mkdir_rr=true + + getfattr -d -m dmv $testdir | grep dmv + if [ $? -eq 0 ] && [ $MDS1_VERSION -ge $(version_code 2.14.51) ]; then + local inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir) + + (( $inherit_rr == 0 )) && test_mkdir_rr=false + fi + + echo + $test_mkdir_rr && + echo "Mkdir (stripe_count $stripe_count) roundrobin:" || + echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index" + for i in $(seq $((100 * MDSCOUNT))); do eval $mkdir_cmd $testdir/subdir$i || error "$mkdir_cmd subdir$i failed" @@ -23772,15 +23784,24 @@ test_qos_mkdir() { count=$($LFS getdirstripe -i $testdir/* | grep ^$((i - 1))$ | wc -l) echo "$count directories created on MDT$((i - 1))" - [ $count -eq 100 ] || error "subdirs are not evenly distributed" + if $test_mkdir_rr; then + (( $count == 100 )) || + error "subdirs are not evenly distributed" + elif [ $((i - 1)) -eq $stripe_index ]; then + (( $count == 100 * MDSCOUNT )) || + error "$count subdirs created on MDT$((i - 1))" + else + (( $count == 0 )) || + error "$count subdirs created on MDT$((i - 1))" + fi - if [ $stripe_count -gt 1 ]; then + if $test_mkdir_rr && [ $stripe_count -gt 1 ]; then count=$($LFS getdirstripe $testdir/* | grep -P "^\s+$((i - 1))\t" | wc -l) echo "$count stripes created on MDT$((i - 1))" # deviation should < 5% of average - [ $count -lt $((95 * stripe_count)) ] || - [ $count -gt $((105 * stripe_count)) ] && + (( $count < 95 * stripe_count )) || + (( $count > 105 * stripe_count)) && error "stripes are not evenly distributed" fi done @@ -23820,10 +23841,10 @@ test_qos_mkdir() { fi done - [ ${ffree[min_index]} -eq 0 ] && + (( ${ffree[min_index]} == 0 )) && skip "no free files in MDT$min_index" - [ ${ffree[min_index]} -gt 100000000 ] && - skip "too much free files in MDT$min_index" + (( ${ffree[min_index]} > 100000000 )) && + skip "too many free files in MDT$min_index" # Check if we need to generate uneven MDTs local threshold=50 @@ -23835,20 +23856,14 @@ test_qos_mkdir() { echo -n "weight diff=$diff% must be > $threshold% ..." count=$((${ffree[min_index]} / 10)) # 50 sec per 10000 files in vm - [ $count -gt 40000 ] && [ "$SLOW" = "no" ] && + (( $count < 100000 )) || [ "$SLOW" != "no" ] || skip "$count files to create" echo "Fill MDT$min_index with $count files" [ -d $DIR/$tdir-MDT$min_index ] || $LFS mkdir -i $min_index $DIR/$tdir-MDT$min_index || error "mkdir $tdir-MDT$min_index failed" - for i in $(seq $count); do - $OPENFILE -f O_CREAT:O_LOV_DELAY_CREATE \ - $DIR/$tdir-MDT$min_index/f$j_$i > /dev/null || - error "create f$j_$i failed" - setfattr -n user.413b -v $value \ - $DIR/$tdir-MDT$min_index/f$j_$i || - error "setfattr f$j_$i failed" - done + createmany -d $DIR/$tdir-MDT$min_index/d $count || + error "create d$count failed" ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-*.filesfree)) bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-*.kbytesavail)) @@ -23897,7 +23912,7 @@ test_qos_mkdir() { min=$($LFS getdirstripe -i $testdir/* | grep ^$min_index$ | wc -l) # D-value should > 10% of averge - [ $((max - min)) -lt 10 ] && + (( $max - $min < 10 )) && error "subdirs shouldn't be evenly distributed" # ditto @@ -23906,7 +23921,7 @@ test_qos_mkdir() { grep -P "^\s+$max_index\t" | wc -l) min=$($LFS getdirstripe $testdir/* | grep -P "^\s+$min_index\t" | wc -l) - [ $((max - min)) -le $((10 * stripe_count)) ] && + (( $max - $min < 10 * $stripe_count )) && error "stripes shouldn't be evenly distributed"|| true fi } @@ -23936,23 +23951,64 @@ test_413b() { [ $MDS1_VERSION -lt $(version_code 2.12.52) ] && skip "Need server version at least 2.12.52" + local testdir local stripe_count for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do - mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed" - mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed" - mkdir $DIR/$tdir-s$stripe_count/qos || error "mkdir failed" - $LFS setdirstripe -D -c $stripe_count \ - $DIR/$tdir-s$stripe_count/rr || - error "setdirstripe failed" - $LFS setdirstripe -D -c $stripe_count \ - $DIR/$tdir-s$stripe_count/qos || + testdir=$DIR/$tdir-s$stripe_count + mkdir $testdir || error "mkdir $testdir failed" + mkdir $testdir/rr || error "mkdir rr failed" + mkdir $testdir/qos || error "mkdir qos failed" + $LFS setdirstripe -D -c $stripe_count --max-inherit-rr 2 \ + $testdir/rr || error "setdirstripe rr failed" + $LFS setdirstripe -D -c $stripe_count $testdir/qos || error "setdirstripe failed" test_qos_mkdir "mkdir" $stripe_count done } run_test 413b "QoS mkdir under dir whose default LMV starting MDT offset is -1" +test_413c() { + [ $MDSCOUNT -ge 2 ] || + skip "We need at least 2 MDTs for this test" + + [ $MDS1_VERSION -ge $(version_code 2.14.51) ] || + skip "Need server version at least 2.14.50" + + local testdir + local inherit + local inherit_rr + + testdir=$DIR/${tdir}-s1 + mkdir $testdir || error "mkdir $testdir failed" + mkdir $testdir/rr || error "mkdir rr failed" + mkdir $testdir/qos || error "mkdir qos failed" + # default max_inherit is -1, default max_inherit_rr is 0 + $LFS setdirstripe -D -c 1 $testdir/rr || + error "setdirstripe rr failed" + $LFS setdirstripe -D -c 1 -X 2 --max-inherit-rr 1 $testdir/qos || + error "setdirstripe qos failed" + test_qos_mkdir "mkdir" 1 + + mkdir $testdir/rr/level1 || error "mkdir rr/level1 failed" + inherit=$($LFS getdirstripe -D -X $testdir/rr/level1) + (( $inherit == -1 )) || error "rr/level1 inherit $inherit != -1" + inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir/rr/level1) + (( $inherit_rr == 0 )) || + error "rr/level1 inherit-rr $inherit_rr != 0" + + mkdir $testdir/qos/level1 || error "mkdir qos/level1 failed" + inherit=$($LFS getdirstripe -D -X $testdir/qos/level1) + (( $inherit == 1 )) || error "qos/level1 inherit $inherit != 1" + inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir/qos/level1) + (( $inherit_rr == 0 )) || + error "qos/level1 inherit-rr $inherit_rr !=0" + mkdir $testdir/qos/level1/level2 || error "mkdir level2 failed" + getfattr -d -m dmv $testdir/qos/level1/level2 | grep dmv && + error "level2 shouldn't have default LMV" || true +} +run_test 413c "mkdir with default LMV max inherit rr" + test_414() { #define OBD_FAIL_PTLRPC_BULK_ATTACH 0x521 $LCTL set_param fail_loc=0x80000521 diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index ca8304f..77aabd2 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -287,7 +287,9 @@ static inline int lfs_mirror_delete(int argc, char **argv) " [--mdt-count|-c stripe_count>\n" \ " [--mdt-hash|-H mdt_hash]\n" \ " [--mdt-index|-i mdt_index[,mdt_index,...]\n" \ - " [--default|-D] [--mode|-o mode] \n" \ + " [--default|-D] [--mode|-o mode]\n" \ + " [--max-inherit|-X max_inherit]\n" \ + " [--max-inherit-rr max_inherit_rr] \n" \ "\tstripe_count: stripe count of the striped directory\n" \ "\tmdt_index: MDT index of first stripe\n" \ "\tmdt_hash: hash type of the striped directory. mdt types:\n" \ @@ -462,7 +464,9 @@ command_t cmdlist[] = { "usage: getdirstripe [--mdt-count|-c] [--mdt-index|-m|-i]\n" " [--mdt-hash|-H] [--obd|-O ]\n" " [--recursive|-r] [--yaml|-y]\n" - " [--verbose|-v] [--default|-D] ..."}, + " [--verbose|-v] [--default|-D]\n" + " [--max-inherit|-X]\n" + " [--max-inherit-rr] ..."}, {"mkdir", lfs_setdirstripe, 0, "Create striped directory on specified MDT, same as setdirstripe.\n" "usage: mkdir [OPTION] \n" @@ -3347,6 +3351,7 @@ enum { LFS_LAYOUT_FOREIGN_OPT, LFS_MODE_OPT, LFS_NEWERXY_OPT, + LFS_INHERIT_RR_OPT, }; /* functions */ @@ -3481,6 +3486,7 @@ static int lfs_setstripe_internal(int argc, char **argv, /* --verbose is only valid in migrate mode */ { .val = 'v', .name = "verbose", .has_arg = no_argument}, { .val = 'x', .name = "xattr", .has_arg = required_argument }, +/* dirstripe { .val = 'X',.name = "max-inherit",.has_arg = required_argument }*/ { .val = 'y', .name = "yaml", .has_arg = required_argument }, { .val = 'z', .name = "ext-size", .has_arg = required_argument}, { .val = 'z', .name = "extension-size", .has_arg = required_argument}, @@ -5525,6 +5531,7 @@ static int lfs_getstripe_internal(int argc, char **argv, /* find { .val = 'u', .name = "uid", .has_arg = required_argument }*/ /* find { .val = 'U', .name = "user", .has_arg = required_argument }*/ { .val = 'v', .name = "verbose", .has_arg = no_argument }, +/* dirstripe { .val = 'X',.name = "max-inherit",.has_arg = required_argument }*/ { .val = 'y', .name = "yaml", .has_arg = no_argument }, { .val = 'z', .name = "extension-size", .has_arg = no_argument }, { .val = 'z', .name = "ext-size", .has_arg = no_argument }, @@ -5907,23 +5914,26 @@ static int lfs_getdirstripe(int argc, char **argv) { struct find_param param = { 0 }; struct option long_opts[] = { - { .val = 'c', .name = "mdt-count", .has_arg = no_argument }, - { .val = 'D', .name = "default", .has_arg = no_argument }, - { .val = 'H', .name = "mdt-hash", .has_arg = no_argument }, - { .val = 'i', .name = "mdt-index", .has_arg = no_argument }, - { .val = 'm', .name = "mdt-index", .has_arg = no_argument }, - { .val = 'O', .name = "obd", .has_arg = required_argument }, - { .val = 'r', .name = "recursive", .has_arg = no_argument }, - { .val = 'T', .name = "mdt-count", .has_arg = no_argument }, - { .val = 'v', .name = "verbose", .has_arg = no_argument }, - { .val = 'y', .name = "yaml", .has_arg = no_argument }, + { .val = 'c', .name = "mdt-count", .has_arg = no_argument }, + { .val = 'D', .name = "default", .has_arg = no_argument }, + { .val = 'H', .name = "mdt-hash", .has_arg = no_argument }, + { .val = 'i', .name = "mdt-index", .has_arg = no_argument }, + { .val = 'm', .name = "mdt-index", .has_arg = no_argument }, + { .val = 'O', .name = "obd", .has_arg = required_argument }, + { .val = 'r', .name = "recursive", .has_arg = no_argument }, + { .val = 'T', .name = "mdt-count", .has_arg = no_argument }, + { .val = 'v', .name = "verbose", .has_arg = no_argument }, + { .val = 'X', .name = "max-inherit", .has_arg = no_argument }, + { .val = 'y', .name = "yaml", .has_arg = no_argument }, + { .val = LFS_INHERIT_RR_OPT, + .name = "max-inherit-rr", .has_arg = no_argument }, { .name = NULL } }; int c, rc; param.fp_get_lmv = 1; while ((c = getopt_long(argc, argv, - "cDHimO:rtTvy", long_opts, NULL)) != -1) { + "cDHimO:rtTvXy", long_opts, NULL)) != -1) { switch (c) { case 'c': case 'T': @@ -5959,6 +5969,12 @@ static int lfs_getdirstripe(int argc, char **argv) case 'v': param.fp_verbose |= VERBOSE_DETAIL; break; + case 'X': + param.fp_verbose |= VERBOSE_INHERIT; + break; + case LFS_INHERIT_RR_OPT: + param.fp_verbose |= VERBOSE_INHERIT_RR; + break; case 'y': param.fp_yaml = 1; break; @@ -6323,6 +6339,10 @@ static int mntdf(char *mntdir, char *fsname, char *pool, enum mntdf_flags flags, return rc; } +enum { + LAYOUT_INHERIT_UNSET = -2, +}; + /* functions */ static int lfs_setdirstripe(int argc, char **argv) { @@ -6340,6 +6360,8 @@ static int lfs_setdirstripe(int argc, char **argv) mode_t previous_mode = 0; char *xattr = NULL; __u32 type = LU_FOREIGN_TYPE_SYMLINK, flags = 0; + int max_inherit = LAYOUT_INHERIT_UNSET; + int max_inherit_rr = LAYOUT_INHERIT_UNSET; struct option long_opts[] = { { .val = 'c', .name = "count", .has_arg = required_argument }, { .val = 'c', .name = "mdt-count", .has_arg = required_argument }, @@ -6367,14 +6389,17 @@ static int lfs_setdirstripe(int argc, char **argv) { .val = 't', .name = "hash-type", .has_arg = required_argument }, #endif { .val = 'T', .name = "mdt-count", .has_arg = required_argument }, -/* setstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */ { .val = 'x', .name = "xattr", .has_arg = required_argument }, + { .val = 'X', .name = "max-inherit", .has_arg = required_argument }, + { .val = LFS_INHERIT_RR_OPT, + .name = "max-inherit-rr", .has_arg = required_argument}, +/* setstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */ { .name = NULL } }; int result = 0; setstripe_args_init(&lsa); - while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:x:", + while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:x:X:", long_opts, NULL)) >= 0) { switch (c) { case 0: @@ -6489,6 +6514,60 @@ static int lfs_setdirstripe(int argc, char **argv) case 'x': xattr = optarg; break; + case 'X': + if (!default_stripe) { + fprintf(stderr, + "%s %s: '--max-inherit' must be specified with '-D'\n", + progname, argv[0]); + return CMD_HELP; + } + errno = 0; + max_inherit = strtol(optarg, &end, 10); + if (errno != 0 || *end != '\0' || max_inherit < -2) { + fprintf(stderr, + "%s %s: invalid max-inherit '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } + if (max_inherit == 0) { + max_inherit = LMV_INHERIT_NONE; + } else if (max_inherit == -1) { + max_inherit = LMV_INHERIT_UNLIMITED; + } else if (max_inherit > LMV_INHERIT_MAX) { + fprintf(stderr, + "%s %s: max-inherit %d exceeds maximum %u\n", + progname, argv[0], max_inherit, + LMV_INHERIT_MAX); + return CMD_HELP; + } + break; + case LFS_INHERIT_RR_OPT: + if (!default_stripe) { + fprintf(stderr, + "%s %s: '--max-inherit-rr' must be specified with '-D'\n", + progname, argv[0]); + return CMD_HELP; + } + errno = 0; + max_inherit_rr = strtol(optarg, &end, 10); + if (errno != 0 || *end != '\0' || max_inherit_rr < -2) { + fprintf(stderr, + "%s %s: invalid max-inherit-rr '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } + if (max_inherit_rr == 0) { + max_inherit_rr = LMV_INHERIT_RR_NONE; + } else if (max_inherit_rr == -1) { + max_inherit_rr = LMV_INHERIT_RR_UNLIMITED; + } else if (max_inherit_rr > LMV_INHERIT_RR_MAX) { + fprintf(stderr, + "%s %s: max-inherit-rr %d exceeds maximum %u\n", + progname, argv[0], max_inherit_rr, + LMV_INHERIT_RR_MAX); + return CMD_HELP; + } + break; default: fprintf(stderr, "%s %s: unrecognized option '%s'\n", progname, argv[0], argv[optind - 1]); @@ -6552,6 +6631,15 @@ static int lfs_setdirstripe(int argc, char **argv) previous_mode = umask(0); } + if (max_inherit_rr != LAYOUT_INHERIT_UNSET && + lsa.lsa_stripe_off != LLAPI_LAYOUT_DEFAULT && + lsa.lsa_stripe_off != -1) { + fprintf(stderr, + "%s %s: max-inherit-rr is meaningless if stripe offset != -1\n", + progname, argv[0]); + return CMD_HELP; + } + /* foreign LMV/dir case */ if (foreign_mode) { if (argc > optind + 1) { @@ -6602,6 +6690,14 @@ static int lfs_setdirstripe(int argc, char **argv) param->lsp_stripe_pattern = LMV_HASH_TYPE_UNKNOWN; param->lsp_pool = lsa.lsa_pool_name; param->lsp_is_specific = false; + if (default_stripe) { + if (max_inherit == LAYOUT_INHERIT_UNSET) + max_inherit = LMV_INHERIT_DEFAULT; + if (max_inherit_rr == LAYOUT_INHERIT_UNSET) + max_inherit_rr = LMV_INHERIT_RR_DEFAULT; + param->lsp_max_inherit = max_inherit; + param->lsp_max_inherit_rr = max_inherit_rr; + } if (strcmp(argv[0], "mkdir") == 0) param->lsp_is_create = true; if (lsa.lsa_nr_tgts > 1) { diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 8ffd467..2d29c35 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -962,6 +962,8 @@ static inline void param2lmu(struct lmv_user_md *lmu, lmu->lum_stripe_count = param->lsp_stripe_count; lmu->lum_stripe_offset = param->lsp_stripe_offset; lmu->lum_hash_type = param->lsp_stripe_pattern; + lmu->lum_max_inherit = param->lsp_max_inherit; + lmu->lum_max_inherit_rr = param->lsp_max_inherit_rr; if (param->lsp_pool != NULL) strncpy(lmu->lum_pool_name, param->lsp_pool, LOV_MAXPOOLNAME); if (param->lsp_is_specific) { @@ -3215,9 +3217,50 @@ void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, if (flags & LMV_HASH_FLAG_LOST_LMV) llapi_printf(LLAPI_MSG_NORMAL, ",lost_lmv"); - separator = "\n"; + if (verbose & VERBOSE_HASH_TYPE && !yaml) + separator = " "; + else + separator = "\n"; + } + + if ((verbose & VERBOSE_INHERIT) && lum->lum_magic == LMV_USER_MAGIC) { + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); + if (verbose & ~VERBOSE_INHERIT) + llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit: "); + if (lum->lum_max_inherit == LMV_INHERIT_UNLIMITED) + llapi_printf(LLAPI_MSG_NORMAL, "-1"); + else if (lum->lum_max_inherit == LMV_INHERIT_NONE) + llapi_printf(LLAPI_MSG_NORMAL, "0"); + else + llapi_printf(LLAPI_MSG_NORMAL, "%hhu", + lum->lum_max_inherit); + if (verbose & VERBOSE_INHERIT && !yaml) + separator = " "; + else + separator = "\n"; } + if ((verbose & VERBOSE_INHERIT_RR) && + lum->lum_magic == LMV_USER_MAGIC && + lum->lum_stripe_offset == LMV_OFFSET_DEFAULT) { + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); + if (verbose & ~VERBOSE_INHERIT_RR) + llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit_rr: "); + if (lum->lum_max_inherit_rr == LMV_INHERIT_RR_UNLIMITED) + llapi_printf(LLAPI_MSG_NORMAL, "-1"); + else if (lum->lum_max_inherit_rr == LMV_INHERIT_RR_NONE) + llapi_printf(LLAPI_MSG_NORMAL, "0"); + else + llapi_printf(LLAPI_MSG_NORMAL, "%hhu", + lum->lum_max_inherit_rr); + if (verbose & VERBOSE_INHERIT_RR && !yaml) + separator = " "; + else + separator = "\n"; + } + + separator = "\n"; + if (verbose & VERBOSE_OBJID && lum->lum_magic != LMV_USER_MAGIC) { llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); if (lum->lum_stripe_count > 0) -- 1.8.3.1