.SH NAME
lfs getdirstripe \- list the layout pattern of a given directory
.SH SYNOPSIS
-.BR "lfs getdirstripe" " [" -cDimOrTy "]... " \fIDIR\fR...
+.BR "lfs getdirstripe" " [" -cDimOrTXy "]... " \fIDIR\fR...
.SH DESCRIPTION
Get the layout pattern of striped directories. This
.BR lfs (1)
.BR --recursive ,
which will recurse into all subdirectories.
.TP
+.BR \-X ", " \-\-max-inherit
+Show the inherit depth of default layout.
+.TP
.BR \-y ", " \-\-yaml
Show the layout in YAML format for easier parsing.
+.TP
+.BR \-\-max-inherit-rr
+Show the inherit depth of default layout to roundrobin mkdir.
.SH EXAMPLES
.TP
.B $ lfs getdirstripe -O lustre-MDT0001_UUID -r /mnt/lustre/dir1
.SH NAME
lfs setdirstripe, mkdir \- set striping pattern of a directory.
.SH SYNOPSIS
-.B lfs setdirstripe [\fR-cdDHioTx\fR] \fIDIR\fR...
+.B lfs setdirstripe [\fR-cdDHioTxX\fR] \fIDIR\fR...
.br
.SH DESCRIPTION
Create a striped directory with specified striping pattern. This
.TP
.BR \-x ", " \-\-xattr =\fISTRING\fR
Specify a string to be used as a foreign (free format) striping.
+.TP
+.BR \-X ", " \-\-max-inherit = \fIMAX_INHERIT
+Set the inherit depth of default directory layout. If non-zero, then
+.I MAX_INHERIT
+is the number of subdirectory levels for which this default layout is inherited,
+up to a maximum of 250 levels, and is decremented by one when copying the
+default layout to each new subdirectory, until zero and the default layout is
+no longer copied. A
+.I MAX_INHERIT
+of -1 means the default layout is inherited for all subdirectories.
+.TP
+.BR \-\-max-inherit-rr = \fIMAX_INHERIT_RR
+Set the round-robin inherit depth of the default directory layout, only when
+.I START_MDT_INDEX
+is -1. If
+.I MAX_INHERIT_RR
+is zero (unset), then new subdirectories are preferentially created on MDTs
+with more free space and inodes if the MDTs are imbalanced. If non-zero, then
+.I MAX_INHERIT_RR
+is the number of subdirectory levels, up to a maximum of 250 levels, for which
+new subdirectories will be created in a round-robin manner across all available
+MDTs, rather than using MDT space balancing for new subdirectories. Otherwise,
+.I MAX_INHERIT_RR
+is decremented by one when copying the default layout to each new subdirectory,
+until zero and round-robin is no longer used. It may be useful to set
+.I MAX_INHERIT_RR
+to 2 or 3 when setting the default directory layout on the root of a
+.B new
+filesystem, so that top-level subdirectories are immediately spread
+across MDTs, rather than waiting for the MDTs to become imbalanced.
+Round-robin subdirectory creation is unlikely to be useful for existing
+filesystems that already have many files and imbalanced MDTs, since space
+balancing will already happen, and this will unnecessarily increase the number
+of remote subdirectories (increasing overhead) without any benefit.
.SH NOTE
.PP
If neither
void lu_tgt_descs_fini(struct lu_tgt_descs *ltd);
int ltd_add_tgt(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt);
void ltd_del_tgt(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt);
-bool ltd_qos_is_usable(struct lu_tgt_descs *ltd);
int ltd_qos_penalties_calc(struct lu_tgt_descs *ltd);
int ltd_qos_update(struct lu_tgt_descs *ltd, struct lu_tgt_desc *tgt,
__u64 *total_wt);
+/**
+ * Whether MDT inode and space usages are balanced.
+ */
+static inline bool ltd_qos_is_balanced(struct lu_tgt_descs *ltd)
+{
+ return !test_bit(LQ_DIRTY, <d->ltd_qos.lq_flags) &&
+ test_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags);
+}
+
+/**
+ * Whether QoS data is up-to-date and QoS can be applied.
+ */
+static inline bool ltd_qos_is_usable(struct lu_tgt_descs *ltd)
+{
+ if (ltd_qos_is_balanced(ltd))
+ return false;
+
+ if (ltd->ltd_lov_desc.ld_active_tgt_count < 2)
+ return false;
+
+ return true;
+}
+
static inline struct lu_tgt_desc *ltd_first_tgt(struct lu_tgt_descs *ltd)
{
int index;
int lsp_stripe_count;
bool lsp_is_specific;
bool lsp_is_create;
+ __u8 lsp_max_inherit;
+ __u8 lsp_max_inherit_rr;
__u32 lsp_osts[0];
};
void llapi_clear_command_name(void);
enum llapi_layout_verbose {
- VERBOSE_STRIPE_COUNT = 0x1,
- VERBOSE_STRIPE_SIZE = 0x2,
- VERBOSE_STRIPE_OFFSET = 0x4,
- VERBOSE_POOL = 0x8,
- VERBOSE_DETAIL = 0x10,
- VERBOSE_OBJID = 0x20,
- VERBOSE_GENERATION = 0x40,
- VERBOSE_MDTINDEX = 0x80,
- VERBOSE_PATTERN = 0x100,
- VERBOSE_COMP_COUNT = 0x200,
- VERBOSE_COMP_FLAGS = 0x400,
- VERBOSE_COMP_START = 0x800,
- VERBOSE_COMP_END = 0x1000,
- VERBOSE_COMP_ID = 0x2000,
- VERBOSE_DFID = 0x4000,
- VERBOSE_HASH_TYPE = 0x8000,
- VERBOSE_MIRROR_COUNT = 0x10000,
- VERBOSE_MIRROR_ID = 0x20000,
- VERBOSE_EXT_SIZE = 0x40000,
+ VERBOSE_STRIPE_COUNT = 0x1,
+ VERBOSE_STRIPE_SIZE = 0x2,
+ VERBOSE_STRIPE_OFFSET = 0x4,
+ VERBOSE_POOL = 0x8,
+ VERBOSE_DETAIL = 0x10,
+ VERBOSE_OBJID = 0x20,
+ VERBOSE_GENERATION = 0x40,
+ VERBOSE_MDTINDEX = 0x80,
+ VERBOSE_PATTERN = 0x100,
+ VERBOSE_COMP_COUNT = 0x200,
+ VERBOSE_COMP_FLAGS = 0x400,
+ VERBOSE_COMP_START = 0x800,
+ VERBOSE_COMP_END = 0x1000,
+ VERBOSE_COMP_ID = 0x2000,
+ VERBOSE_DFID = 0x4000,
+ VERBOSE_HASH_TYPE = 0x8000,
+ VERBOSE_MIRROR_COUNT = 0x10000,
+ VERBOSE_MIRROR_ID = 0x20000,
+ VERBOSE_EXT_SIZE = 0x40000,
+ VERBOSE_INHERIT = 0x80000,
+ VERBOSE_INHERIT_RR = 0x100000,
VERBOSE_DEFAULT = VERBOSE_STRIPE_COUNT | VERBOSE_STRIPE_SIZE |
VERBOSE_STRIPE_OFFSET | VERBOSE_POOL |
VERBOSE_OBJID | VERBOSE_GENERATION |
VERBOSE_COMP_COUNT | VERBOSE_COMP_FLAGS |
VERBOSE_COMP_START | VERBOSE_COMP_END |
VERBOSE_COMP_ID | VERBOSE_MIRROR_COUNT |
- VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE
+ VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE |
+ VERBOSE_INHERIT | VERBOSE_INHERIT_RR
};
/* Compatibility with original names */
#define VERBOSE_SIZE VERBOSE_STRIPE_SIZE
__u32 lsm_md_stripe_count;
__u32 lsm_md_master_mdt_index;
__u32 lsm_md_hash_type;
+ __u8 lsm_md_max_inherit;
+ __u8 lsm_md_max_inherit_rr;
__u32 lsm_md_layout_version;
__u32 lsm_md_migrate_offset;
__u32 lsm_md_migrate_hash;
* terminated string so only print LOV_MAXPOOLNAME bytes.
*/
CDEBUG(mask,
- "magic %#x stripe count %d master mdt %d hash type %#x version %d migrate offset %d migrate hash %#x pool %.*s\n",
+ "magic %#x stripe count %d master mdt %d hash type %#x max inherit %hhu version %d migrate offset %d migrate hash %#x pool %.*s\n",
lsm->lsm_md_magic, lsm->lsm_md_stripe_count,
lsm->lsm_md_master_mdt_index, lsm->lsm_md_hash_type,
- lsm->lsm_md_layout_version, lsm->lsm_md_migrate_offset,
- lsm->lsm_md_migrate_hash,
+ lsm->lsm_md_max_inherit, lsm->lsm_md_layout_version,
+ lsm->lsm_md_migrate_offset, lsm->lsm_md_migrate_hash,
LOV_MAXPOOLNAME, lsm->lsm_md_pool_name);
if (!lmv_dir_striped(lsm))
#define LMV_MAX_STRIPE_COUNT 2000 /* ((12 * 4096 - 256) / 24) */
#define lmv_user_md lmv_user_md_v1
struct lmv_user_md_v1 {
- __u32 lum_magic; /* must be the first field */
+ __u32 lum_magic; /* must be the first field */
__u32 lum_stripe_count; /* dirstripe count */
__u32 lum_stripe_offset; /* MDT idx for default dirstripe */
__u32 lum_hash_type; /* Dir stripe policy */
- __u32 lum_type; /* LMV type: default */
- __u32 lum_padding1;
+ __u32 lum_type; /* LMV type: default */
+ __u8 lum_max_inherit; /* inherit depth of default LMV */
+ __u8 lum_max_inherit_rr; /* inherit depth of default LMV to round-robin mkdir */
+ __u16 lum_padding1;
__u32 lum_padding2;
__u32 lum_padding3;
char lum_pool_name[LOV_MAXPOOLNAME + 1];
LMV_TYPE_DEFAULT = 0x0000,
};
+/* lum_max_inherit will be decreased by 1 after each inheritance if it's not
+ * LMV_INHERIT_UNLIMITED or > LMV_INHERIT_MAX.
+ */
+enum {
+ /* for historical reason, 0 means unlimited inheritance */
+ LMV_INHERIT_UNLIMITED = 0,
+ /* unlimited lum_max_inherit by default */
+ LMV_INHERIT_DEFAULT = 0,
+ /* not inherit any more */
+ LMV_INHERIT_END = 1,
+ /* max inherit depth */
+ LMV_INHERIT_MAX = 250,
+ /* [251, 254] are reserved */
+ /* not set, or when inherit depth goes beyond end, */
+ LMV_INHERIT_NONE = 255,
+};
+
+enum {
+ /* not set, or when inherit_rr depth goes beyond end, */
+ LMV_INHERIT_RR_NONE = 0,
+ /* disable lum_max_inherit_rr by default */
+ LMV_INHERIT_RR_DEFAULT = 0,
+ /* not inherit any more */
+ LMV_INHERIT_RR_END = 1,
+ /* max inherit depth */
+ LMV_INHERIT_RR_MAX = 250,
+ /* [251, 254] are reserved */
+ /* unlimited inheritance */
+ LMV_INHERIT_RR_UNLIMITED = 255,
+};
+
static inline int lmv_user_md_size(int stripes, int lmm_magic)
{
int size = sizeof(struct lmv_user_md);
md.default_lmv->lsm_md_master_mdt_index =
lum->lum_stripe_offset;
md.default_lmv->lsm_md_hash_type = lum->lum_hash_type;
+ md.default_lmv->lsm_md_max_inherit =
+ lum->lum_max_inherit;
+ md.default_lmv->lsm_md_max_inherit_rr =
+ lum->lum_max_inherit_rr;
err = ll_update_inode(dir, &md);
md_free_lustre_md(sbi->ll_md_exp, &md);
return rc;
}
+static inline bool lmv_op_user_qos_mkdir(const struct md_op_data *op_data)
+{
+ const struct lmv_user_md *lum = op_data->op_data;
+
+ return (op_data->op_cli_flags & CLI_SET_MEA) && lum &&
+ le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC &&
+ le32_to_cpu(lum->lum_stripe_offset) == LMV_OFFSET_DEFAULT;
+}
+
+static inline bool lmv_op_default_qos_mkdir(const struct md_op_data *op_data)
+{
+ const struct lmv_stripe_md *lsm = op_data->op_default_mea1;
+
+ return lsm && lsm->lsm_md_master_mdt_index == LMV_OFFSET_DEFAULT;
+}
+
/* mkdir by QoS in two cases:
* 1. 'lfs mkdir -i -1'
* 2. parent default LMV master_mdt_index is -1
*/
static inline bool lmv_op_qos_mkdir(const struct md_op_data *op_data)
{
- const struct lmv_stripe_md *lsm = op_data->op_default_mea1;
- const struct lmv_user_md *lum = op_data->op_data;
-
if (op_data->op_code != LUSTRE_OPC_MKDIR)
return false;
if (lmv_dir_striped(op_data->op_mea1))
return false;
- if (op_data->op_cli_flags & CLI_SET_MEA && lum &&
- (le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC ||
- le32_to_cpu(lum->lum_magic) == LMV_USER_MAGIC_SPECIFIC) &&
- le32_to_cpu(lum->lum_stripe_offset) == LMV_OFFSET_DEFAULT)
+ if (lmv_op_user_qos_mkdir(op_data))
return true;
- if (lsm && lsm->lsm_md_master_mdt_index == LMV_OFFSET_DEFAULT)
+ if (lmv_op_default_qos_mkdir(op_data))
return true;
return false;
}
+/* if default LMV is set, and its index is LMV_OFFSET_DEFAULT, and
+ * 1. max_inherit_rr is set and is not LMV_INHERIT_RR_NONE
+ * 2. or parent is ROOT
+ * mkdir roundrobin.
+ * NB, this also needs to check server is balanced, which is checked by caller.
+ */
+static inline bool lmv_op_default_rr_mkdir(const struct md_op_data *op_data)
+{
+ const struct lmv_stripe_md *lsm = op_data->op_default_mea1;
+
+ if (!lmv_op_default_qos_mkdir(op_data))
+ return false;
+
+ return lsm->lsm_md_max_inherit_rr != LMV_INHERIT_RR_NONE ||
+ fid_is_root(&op_data->op_fid1);
+}
+
/* 'lfs mkdir -i <specific_MDT>' */
static inline bool lmv_op_user_specific_mkdir(const struct md_op_data *op_data)
{
op_data->op_default_mea1->lsm_md_master_mdt_index !=
LMV_OFFSET_DEFAULT;
}
+
int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
const void *data, size_t datalen, umode_t mode, uid_t uid,
gid_t gid, cfs_cap_t cap_effective, __u64 rdev,
if (!tgt)
RETURN(-ENODEV);
} else if (lmv_op_qos_mkdir(op_data)) {
+ struct lmv_tgt_desc *tmp = tgt;
+
tgt = lmv_locate_tgt_qos(lmv, &op_data->op_mds);
- if (tgt == ERR_PTR(-EAGAIN))
- tgt = lmv_locate_tgt_rr(lmv, &op_data->op_mds);
+ if (tgt == ERR_PTR(-EAGAIN)) {
+ if (ltd_qos_is_balanced(&lmv->lmv_mdt_descs) &&
+ !lmv_op_default_rr_mkdir(op_data) &&
+ !lmv_op_user_qos_mkdir(op_data))
+ /* if it's not necessary, don't create remote
+ * directory.
+ */
+ tgt = tmp;
+ else
+ tgt = lmv_locate_tgt_rr(lmv, &op_data->op_mds);
+ }
if (IS_ERR(tgt))
RETURN(PTR_ERR(tgt));
+
/*
* only update statfs after QoS mkdir, this means the cached
* statfs may be stale, and current mkdir may not follow QoS
lsm->lsm_md_stripe_count = le32_to_cpu(lmu->lum_stripe_count);
lsm->lsm_md_master_mdt_index = le32_to_cpu(lmu->lum_stripe_offset);
lsm->lsm_md_hash_type = le32_to_cpu(lmu->lum_hash_type);
+ lsm->lsm_md_max_inherit = lmu->lum_max_inherit;
+ lsm->lsm_md_max_inherit_rr = lmu->lum_max_inherit_rr;
lsm->lsm_md_pool_name[LOV_MAXPOOLNAME] = 0;
return 0;
__u32 lds_dir_def_stripe_count;
__u32 lds_dir_def_stripe_offset;
__u32 lds_dir_def_hash_type;
+ __u8 lds_dir_def_max_inherit;
+ __u8 lds_dir_def_max_inherit_rr;
/* default file striping flags (LOV) */
__u32 lds_def_striping_set:1,
lds_def_striping_is_composite:1,
lds_dir_def_striping_set:1;
};
+static inline __u8 lmv_inherit_next(__u8 inherit)
+{
+ if (inherit == LMV_INHERIT_END || inherit == LMV_INHERIT_NONE)
+ return LMV_INHERIT_NONE;
+
+ if (inherit == LMV_INHERIT_UNLIMITED || inherit > LMV_INHERIT_MAX)
+ return inherit;
+
+ return inherit - 1;
+}
+
+static inline __u8 lmv_inherit_rr_next(__u8 inherit_rr)
+{
+ if (inherit_rr == LMV_INHERIT_RR_NONE ||
+ inherit_rr == LMV_INHERIT_RR_UNLIMITED ||
+ inherit_rr > LMV_INHERIT_RR_MAX)
+ return inherit_rr;
+
+ return inherit_rr - 1;
+}
+
struct lod_mirror_entry {
__u16 lme_stale:1,
lme_prefer:1;
/* Transfer default LMV striping from the parent */
if (lds != NULL && lds->lds_dir_def_striping_set &&
+ lds->lds_dir_def_max_inherit != LMV_INHERIT_END &&
+ lds->lds_dir_def_max_inherit != LMV_INHERIT_NONE &&
!(LMVEA_DELETE_VALUES(lds->lds_dir_def_stripe_count,
lds->lds_dir_def_stripe_offset) &&
le32_to_cpu(lds->lds_dir_def_hash_type) !=
cpu_to_le32(lds->lds_dir_def_stripe_offset);
v1->lum_hash_type =
cpu_to_le32(lds->lds_dir_def_hash_type);
+ v1->lum_max_inherit =
+ lmv_inherit_next(lds->lds_dir_def_max_inherit);
+ v1->lum_max_inherit_rr =
+ lmv_inherit_rr_next(lds->lds_dir_def_max_inherit_rr);
info->lti_buf.lb_buf = v1;
info->lti_buf.lb_len = sizeof(*v1);
le32_to_cpu(lmu->lum_stripe_offset);
lds->lds_dir_def_hash_type =
le32_to_cpu(lmu->lum_hash_type);
+ lds->lds_dir_def_max_inherit = lmu->lum_max_inherit;
+ lds->lds_dir_def_max_inherit_rr = lmu->lum_max_inherit_rr;
lds->lds_dir_def_striping_set = 1;
}
EXPORT_SYMBOL(ltd_del_tgt);
/**
- * Whether QoS data is up-to-date and QoS can be applied.
- */
-bool ltd_qos_is_usable(struct lu_tgt_descs *ltd)
-{
- if (!test_bit(LQ_DIRTY, <d->ltd_qos.lq_flags) &&
- test_bit(LQ_SAME_SPACE, <d->ltd_qos.lq_flags))
- return false;
-
- if (ltd->ltd_lov_desc.ld_active_tgt_count < 2)
- return false;
-
- return true;
-}
-EXPORT_SYMBOL(ltd_qos_is_usable);
-
-/**
* Calculate penalties per-tgt and per-server
*
* Re-calculate penalties when the configuration changes, active targets
__swab32s(&lum->lum_stripe_offset);
__swab32s(&lum->lum_hash_type);
__swab32s(&lum->lum_type);
+ /* lum_max_inherit and lum_max_inherit_rr do not need to be swabbed */
BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding1) == 0);
+ BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding2) == 0);
+ BUILD_BUG_ON(offsetof(typeof(*lum), lum_padding3) == 0);
switch (lum->lum_magic) {
case LMV_USER_MAGIC_SPECIFIC:
count = lum->lum_stripe_count;
stack_trap "do_nodes $mdts $LCTL set_param \
lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" EXIT
- echo
- echo "Mkdir (stripe_count $stripe_count) roundrobin:"
-
$LCTL set_param lmv.*.qos_threshold_rr=100 > /dev/null
do_nodes $mdts $LCTL set_param lod.*.mdt_qos_threshold_rr=100 > /dev/null
testdir=$DIR/$tdir-s$stripe_count/rr
+ local stripe_index=$($LFS getstripe -m $testdir)
+ local test_mkdir_rr=true
+
+ getfattr -d -m dmv $testdir | grep dmv
+ if [ $? -eq 0 ] && [ $MDS1_VERSION -ge $(version_code 2.14.51) ]; then
+ local inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir)
+
+ (( $inherit_rr == 0 )) && test_mkdir_rr=false
+ fi
+
+ echo
+ $test_mkdir_rr &&
+ echo "Mkdir (stripe_count $stripe_count) roundrobin:" ||
+ echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index"
+
for i in $(seq $((100 * MDSCOUNT))); do
eval $mkdir_cmd $testdir/subdir$i ||
error "$mkdir_cmd subdir$i failed"
count=$($LFS getdirstripe -i $testdir/* |
grep ^$((i - 1))$ | wc -l)
echo "$count directories created on MDT$((i - 1))"
- [ $count -eq 100 ] || error "subdirs are not evenly distributed"
+ if $test_mkdir_rr; then
+ (( $count == 100 )) ||
+ error "subdirs are not evenly distributed"
+ elif [ $((i - 1)) -eq $stripe_index ]; then
+ (( $count == 100 * MDSCOUNT )) ||
+ error "$count subdirs created on MDT$((i - 1))"
+ else
+ (( $count == 0 )) ||
+ error "$count subdirs created on MDT$((i - 1))"
+ fi
- if [ $stripe_count -gt 1 ]; then
+ if $test_mkdir_rr && [ $stripe_count -gt 1 ]; then
count=$($LFS getdirstripe $testdir/* |
grep -P "^\s+$((i - 1))\t" | wc -l)
echo "$count stripes created on MDT$((i - 1))"
# deviation should < 5% of average
- [ $count -lt $((95 * stripe_count)) ] ||
- [ $count -gt $((105 * stripe_count)) ] &&
+ (( $count < 95 * stripe_count )) ||
+ (( $count > 105 * stripe_count)) &&
error "stripes are not evenly distributed"
fi
done
fi
done
- [ ${ffree[min_index]} -eq 0 ] &&
+ (( ${ffree[min_index]} == 0 )) &&
skip "no free files in MDT$min_index"
- [ ${ffree[min_index]} -gt 100000000 ] &&
- skip "too much free files in MDT$min_index"
+ (( ${ffree[min_index]} > 100000000 )) &&
+ skip "too many free files in MDT$min_index"
# Check if we need to generate uneven MDTs
local threshold=50
echo -n "weight diff=$diff% must be > $threshold% ..."
count=$((${ffree[min_index]} / 10))
# 50 sec per 10000 files in vm
- [ $count -gt 40000 ] && [ "$SLOW" = "no" ] &&
+ (( $count < 100000 )) || [ "$SLOW" != "no" ] ||
skip "$count files to create"
echo "Fill MDT$min_index with $count files"
[ -d $DIR/$tdir-MDT$min_index ] ||
$LFS mkdir -i $min_index $DIR/$tdir-MDT$min_index ||
error "mkdir $tdir-MDT$min_index failed"
- for i in $(seq $count); do
- $OPENFILE -f O_CREAT:O_LOV_DELAY_CREATE \
- $DIR/$tdir-MDT$min_index/f$j_$i > /dev/null ||
- error "create f$j_$i failed"
- setfattr -n user.413b -v $value \
- $DIR/$tdir-MDT$min_index/f$j_$i ||
- error "setfattr f$j_$i failed"
- done
+ createmany -d $DIR/$tdir-MDT$min_index/d $count ||
+ error "create d$count failed"
ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-*.filesfree))
bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-*.kbytesavail))
min=$($LFS getdirstripe -i $testdir/* | grep ^$min_index$ | wc -l)
# D-value should > 10% of averge
- [ $((max - min)) -lt 10 ] &&
+ (( $max - $min < 10 )) &&
error "subdirs shouldn't be evenly distributed"
# ditto
grep -P "^\s+$max_index\t" | wc -l)
min=$($LFS getdirstripe $testdir/* |
grep -P "^\s+$min_index\t" | wc -l)
- [ $((max - min)) -le $((10 * stripe_count)) ] &&
+ (( $max - $min < 10 * $stripe_count )) &&
error "stripes shouldn't be evenly distributed"|| true
fi
}
[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
skip "Need server version at least 2.12.52"
+ local testdir
local stripe_count
for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
- mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed"
- mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed"
- mkdir $DIR/$tdir-s$stripe_count/qos || error "mkdir failed"
- $LFS setdirstripe -D -c $stripe_count \
- $DIR/$tdir-s$stripe_count/rr ||
- error "setdirstripe failed"
- $LFS setdirstripe -D -c $stripe_count \
- $DIR/$tdir-s$stripe_count/qos ||
+ testdir=$DIR/$tdir-s$stripe_count
+ mkdir $testdir || error "mkdir $testdir failed"
+ mkdir $testdir/rr || error "mkdir rr failed"
+ mkdir $testdir/qos || error "mkdir qos failed"
+ $LFS setdirstripe -D -c $stripe_count --max-inherit-rr 2 \
+ $testdir/rr || error "setdirstripe rr failed"
+ $LFS setdirstripe -D -c $stripe_count $testdir/qos ||
error "setdirstripe failed"
test_qos_mkdir "mkdir" $stripe_count
done
}
run_test 413b "QoS mkdir under dir whose default LMV starting MDT offset is -1"
+test_413c() {
+ [ $MDSCOUNT -ge 2 ] ||
+ skip "We need at least 2 MDTs for this test"
+
+ [ $MDS1_VERSION -ge $(version_code 2.14.51) ] ||
+ skip "Need server version at least 2.14.50"
+
+ local testdir
+ local inherit
+ local inherit_rr
+
+ testdir=$DIR/${tdir}-s1
+ mkdir $testdir || error "mkdir $testdir failed"
+ mkdir $testdir/rr || error "mkdir rr failed"
+ mkdir $testdir/qos || error "mkdir qos failed"
+ # default max_inherit is -1, default max_inherit_rr is 0
+ $LFS setdirstripe -D -c 1 $testdir/rr ||
+ error "setdirstripe rr failed"
+ $LFS setdirstripe -D -c 1 -X 2 --max-inherit-rr 1 $testdir/qos ||
+ error "setdirstripe qos failed"
+ test_qos_mkdir "mkdir" 1
+
+ mkdir $testdir/rr/level1 || error "mkdir rr/level1 failed"
+ inherit=$($LFS getdirstripe -D -X $testdir/rr/level1)
+ (( $inherit == -1 )) || error "rr/level1 inherit $inherit != -1"
+ inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir/rr/level1)
+ (( $inherit_rr == 0 )) ||
+ error "rr/level1 inherit-rr $inherit_rr != 0"
+
+ mkdir $testdir/qos/level1 || error "mkdir qos/level1 failed"
+ inherit=$($LFS getdirstripe -D -X $testdir/qos/level1)
+ (( $inherit == 1 )) || error "qos/level1 inherit $inherit != 1"
+ inherit_rr=$($LFS getdirstripe -D --max-inherit-rr $testdir/qos/level1)
+ (( $inherit_rr == 0 )) ||
+ error "qos/level1 inherit-rr $inherit_rr !=0"
+ mkdir $testdir/qos/level1/level2 || error "mkdir level2 failed"
+ getfattr -d -m dmv $testdir/qos/level1/level2 | grep dmv &&
+ error "level2 shouldn't have default LMV" || true
+}
+run_test 413c "mkdir with default LMV max inherit rr"
+
test_414() {
#define OBD_FAIL_PTLRPC_BULK_ATTACH 0x521
$LCTL set_param fail_loc=0x80000521
" [--mdt-count|-c stripe_count>\n" \
" [--mdt-hash|-H mdt_hash]\n" \
" [--mdt-index|-i mdt_index[,mdt_index,...]\n" \
- " [--default|-D] [--mode|-o mode] <dir>\n" \
+ " [--default|-D] [--mode|-o mode]\n" \
+ " [--max-inherit|-X max_inherit]\n" \
+ " [--max-inherit-rr max_inherit_rr] <dir>\n" \
"\tstripe_count: stripe count of the striped directory\n" \
"\tmdt_index: MDT index of first stripe\n" \
"\tmdt_hash: hash type of the striped directory. mdt types:\n" \
"usage: getdirstripe [--mdt-count|-c] [--mdt-index|-m|-i]\n"
" [--mdt-hash|-H] [--obd|-O <uuid>]\n"
" [--recursive|-r] [--yaml|-y]\n"
- " [--verbose|-v] [--default|-D] <dir> ..."},
+ " [--verbose|-v] [--default|-D]\n"
+ " [--max-inherit|-X]\n"
+ " [--max-inherit-rr] <dir> ..."},
{"mkdir", lfs_setdirstripe, 0,
"Create striped directory on specified MDT, same as setdirstripe.\n"
"usage: mkdir [OPTION] <directory>\n"
LFS_LAYOUT_FOREIGN_OPT,
LFS_MODE_OPT,
LFS_NEWERXY_OPT,
+ LFS_INHERIT_RR_OPT,
};
/* functions */
/* --verbose is only valid in migrate mode */
{ .val = 'v', .name = "verbose", .has_arg = no_argument},
{ .val = 'x', .name = "xattr", .has_arg = required_argument },
+/* dirstripe { .val = 'X',.name = "max-inherit",.has_arg = required_argument }*/
{ .val = 'y', .name = "yaml", .has_arg = required_argument },
{ .val = 'z', .name = "ext-size", .has_arg = required_argument},
{ .val = 'z', .name = "extension-size", .has_arg = required_argument},
/* find { .val = 'u', .name = "uid", .has_arg = required_argument }*/
/* find { .val = 'U', .name = "user", .has_arg = required_argument }*/
{ .val = 'v', .name = "verbose", .has_arg = no_argument },
+/* dirstripe { .val = 'X',.name = "max-inherit",.has_arg = required_argument }*/
{ .val = 'y', .name = "yaml", .has_arg = no_argument },
{ .val = 'z', .name = "extension-size", .has_arg = no_argument },
{ .val = 'z', .name = "ext-size", .has_arg = no_argument },
{
struct find_param param = { 0 };
struct option long_opts[] = {
- { .val = 'c', .name = "mdt-count", .has_arg = no_argument },
- { .val = 'D', .name = "default", .has_arg = no_argument },
- { .val = 'H', .name = "mdt-hash", .has_arg = no_argument },
- { .val = 'i', .name = "mdt-index", .has_arg = no_argument },
- { .val = 'm', .name = "mdt-index", .has_arg = no_argument },
- { .val = 'O', .name = "obd", .has_arg = required_argument },
- { .val = 'r', .name = "recursive", .has_arg = no_argument },
- { .val = 'T', .name = "mdt-count", .has_arg = no_argument },
- { .val = 'v', .name = "verbose", .has_arg = no_argument },
- { .val = 'y', .name = "yaml", .has_arg = no_argument },
+ { .val = 'c', .name = "mdt-count", .has_arg = no_argument },
+ { .val = 'D', .name = "default", .has_arg = no_argument },
+ { .val = 'H', .name = "mdt-hash", .has_arg = no_argument },
+ { .val = 'i', .name = "mdt-index", .has_arg = no_argument },
+ { .val = 'm', .name = "mdt-index", .has_arg = no_argument },
+ { .val = 'O', .name = "obd", .has_arg = required_argument },
+ { .val = 'r', .name = "recursive", .has_arg = no_argument },
+ { .val = 'T', .name = "mdt-count", .has_arg = no_argument },
+ { .val = 'v', .name = "verbose", .has_arg = no_argument },
+ { .val = 'X', .name = "max-inherit", .has_arg = no_argument },
+ { .val = 'y', .name = "yaml", .has_arg = no_argument },
+ { .val = LFS_INHERIT_RR_OPT,
+ .name = "max-inherit-rr", .has_arg = no_argument },
{ .name = NULL } };
int c, rc;
param.fp_get_lmv = 1;
while ((c = getopt_long(argc, argv,
- "cDHimO:rtTvy", long_opts, NULL)) != -1) {
+ "cDHimO:rtTvXy", long_opts, NULL)) != -1) {
switch (c) {
case 'c':
case 'T':
case 'v':
param.fp_verbose |= VERBOSE_DETAIL;
break;
+ case 'X':
+ param.fp_verbose |= VERBOSE_INHERIT;
+ break;
+ case LFS_INHERIT_RR_OPT:
+ param.fp_verbose |= VERBOSE_INHERIT_RR;
+ break;
case 'y':
param.fp_yaml = 1;
break;
return rc;
}
+enum {
+ LAYOUT_INHERIT_UNSET = -2,
+};
+
/* functions */
static int lfs_setdirstripe(int argc, char **argv)
{
mode_t previous_mode = 0;
char *xattr = NULL;
__u32 type = LU_FOREIGN_TYPE_SYMLINK, flags = 0;
+ int max_inherit = LAYOUT_INHERIT_UNSET;
+ int max_inherit_rr = LAYOUT_INHERIT_UNSET;
struct option long_opts[] = {
{ .val = 'c', .name = "count", .has_arg = required_argument },
{ .val = 'c', .name = "mdt-count", .has_arg = required_argument },
{ .val = 't', .name = "hash-type", .has_arg = required_argument },
#endif
{ .val = 'T', .name = "mdt-count", .has_arg = required_argument },
-/* setstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */
{ .val = 'x', .name = "xattr", .has_arg = required_argument },
+ { .val = 'X', .name = "max-inherit", .has_arg = required_argument },
+ { .val = LFS_INHERIT_RR_OPT,
+ .name = "max-inherit-rr", .has_arg = required_argument},
+/* setstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */
{ .name = NULL } };
int result = 0;
setstripe_args_init(&lsa);
- while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:x:",
+ while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:x:X:",
long_opts, NULL)) >= 0) {
switch (c) {
case 0:
case 'x':
xattr = optarg;
break;
+ case 'X':
+ if (!default_stripe) {
+ fprintf(stderr,
+ "%s %s: '--max-inherit' must be specified with '-D'\n",
+ progname, argv[0]);
+ return CMD_HELP;
+ }
+ errno = 0;
+ max_inherit = strtol(optarg, &end, 10);
+ if (errno != 0 || *end != '\0' || max_inherit < -2) {
+ fprintf(stderr,
+ "%s %s: invalid max-inherit '%s'\n",
+ progname, argv[0], optarg);
+ return CMD_HELP;
+ }
+ if (max_inherit == 0) {
+ max_inherit = LMV_INHERIT_NONE;
+ } else if (max_inherit == -1) {
+ max_inherit = LMV_INHERIT_UNLIMITED;
+ } else if (max_inherit > LMV_INHERIT_MAX) {
+ fprintf(stderr,
+ "%s %s: max-inherit %d exceeds maximum %u\n",
+ progname, argv[0], max_inherit,
+ LMV_INHERIT_MAX);
+ return CMD_HELP;
+ }
+ break;
+ case LFS_INHERIT_RR_OPT:
+ if (!default_stripe) {
+ fprintf(stderr,
+ "%s %s: '--max-inherit-rr' must be specified with '-D'\n",
+ progname, argv[0]);
+ return CMD_HELP;
+ }
+ errno = 0;
+ max_inherit_rr = strtol(optarg, &end, 10);
+ if (errno != 0 || *end != '\0' || max_inherit_rr < -2) {
+ fprintf(stderr,
+ "%s %s: invalid max-inherit-rr '%s'\n",
+ progname, argv[0], optarg);
+ return CMD_HELP;
+ }
+ if (max_inherit_rr == 0) {
+ max_inherit_rr = LMV_INHERIT_RR_NONE;
+ } else if (max_inherit_rr == -1) {
+ max_inherit_rr = LMV_INHERIT_RR_UNLIMITED;
+ } else if (max_inherit_rr > LMV_INHERIT_RR_MAX) {
+ fprintf(stderr,
+ "%s %s: max-inherit-rr %d exceeds maximum %u\n",
+ progname, argv[0], max_inherit_rr,
+ LMV_INHERIT_RR_MAX);
+ return CMD_HELP;
+ }
+ break;
default:
fprintf(stderr, "%s %s: unrecognized option '%s'\n",
progname, argv[0], argv[optind - 1]);
previous_mode = umask(0);
}
+ if (max_inherit_rr != LAYOUT_INHERIT_UNSET &&
+ lsa.lsa_stripe_off != LLAPI_LAYOUT_DEFAULT &&
+ lsa.lsa_stripe_off != -1) {
+ fprintf(stderr,
+ "%s %s: max-inherit-rr is meaningless if stripe offset != -1\n",
+ progname, argv[0]);
+ return CMD_HELP;
+ }
+
/* foreign LMV/dir case */
if (foreign_mode) {
if (argc > optind + 1) {
param->lsp_stripe_pattern = LMV_HASH_TYPE_UNKNOWN;
param->lsp_pool = lsa.lsa_pool_name;
param->lsp_is_specific = false;
+ if (default_stripe) {
+ if (max_inherit == LAYOUT_INHERIT_UNSET)
+ max_inherit = LMV_INHERIT_DEFAULT;
+ if (max_inherit_rr == LAYOUT_INHERIT_UNSET)
+ max_inherit_rr = LMV_INHERIT_RR_DEFAULT;
+ param->lsp_max_inherit = max_inherit;
+ param->lsp_max_inherit_rr = max_inherit_rr;
+ }
if (strcmp(argv[0], "mkdir") == 0)
param->lsp_is_create = true;
if (lsa.lsa_nr_tgts > 1) {
lmu->lum_stripe_count = param->lsp_stripe_count;
lmu->lum_stripe_offset = param->lsp_stripe_offset;
lmu->lum_hash_type = param->lsp_stripe_pattern;
+ lmu->lum_max_inherit = param->lsp_max_inherit;
+ lmu->lum_max_inherit_rr = param->lsp_max_inherit_rr;
if (param->lsp_pool != NULL)
strncpy(lmu->lum_pool_name, param->lsp_pool, LOV_MAXPOOLNAME);
if (param->lsp_is_specific) {
if (flags & LMV_HASH_FLAG_LOST_LMV)
llapi_printf(LLAPI_MSG_NORMAL, ",lost_lmv");
- separator = "\n";
+ if (verbose & VERBOSE_HASH_TYPE && !yaml)
+ separator = " ";
+ else
+ separator = "\n";
+ }
+
+ if ((verbose & VERBOSE_INHERIT) && lum->lum_magic == LMV_USER_MAGIC) {
+ llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
+ if (verbose & ~VERBOSE_INHERIT)
+ llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit: ");
+ if (lum->lum_max_inherit == LMV_INHERIT_UNLIMITED)
+ llapi_printf(LLAPI_MSG_NORMAL, "-1");
+ else if (lum->lum_max_inherit == LMV_INHERIT_NONE)
+ llapi_printf(LLAPI_MSG_NORMAL, "0");
+ else
+ llapi_printf(LLAPI_MSG_NORMAL, "%hhu",
+ lum->lum_max_inherit);
+ if (verbose & VERBOSE_INHERIT && !yaml)
+ separator = " ";
+ else
+ separator = "\n";
}
+ if ((verbose & VERBOSE_INHERIT_RR) &&
+ lum->lum_magic == LMV_USER_MAGIC &&
+ lum->lum_stripe_offset == LMV_OFFSET_DEFAULT) {
+ llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
+ if (verbose & ~VERBOSE_INHERIT_RR)
+ llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit_rr: ");
+ if (lum->lum_max_inherit_rr == LMV_INHERIT_RR_UNLIMITED)
+ llapi_printf(LLAPI_MSG_NORMAL, "-1");
+ else if (lum->lum_max_inherit_rr == LMV_INHERIT_RR_NONE)
+ llapi_printf(LLAPI_MSG_NORMAL, "0");
+ else
+ llapi_printf(LLAPI_MSG_NORMAL, "%hhu",
+ lum->lum_max_inherit_rr);
+ if (verbose & VERBOSE_INHERIT_RR && !yaml)
+ separator = " ";
+ else
+ separator = "\n";
+ }
+
+ separator = "\n";
+
if (verbose & VERBOSE_OBJID && lum->lum_magic != LMV_USER_MAGIC) {
llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
if (lum->lum_stripe_count > 0)