From fdad38781cccb05c4cf3f1458c2d2d7c8b2b5bec Mon Sep 17 00:00:00 2001 From: Bruno Faccini Date: Tue, 22 Jan 2019 16:10:26 +0100 Subject: [PATCH] LU-11376 lmv: new foreign LMV format This patch introduces a new striping/LMV format in order to allow to specify an arbitrary external reference for a dir in Lustre namespace. The new LMV format is made of {newmagic, length, type, flags, string[length]} to be as flexible as possible. Foreign dir can be created by using the ioctl(LL_IOC_LMV_SETDIRSTRIPE) operation and it can only be and remain an empty dir until removed. A new API method llapi_dir_create_foreign() has been introduced and "lfs {get,set}dirstripe" and "lfs find" modified to understand new format. The idea behind this is to provide Lustre namespace support and striping prefetch/caching under lock protection, for user/external usage. This patch is the LMV/dirs complement of LOV/files previous change (Change-Id: I5d9c0642fe8e7009c30918bfa946cac7c00c9af8) and has been rebased on top of the latter along with some with obvious mutualizations and simplifications. Code has been added for lfsck to handle foreign dirs, and a new sub-test has been added in sanity-lfsck in order to verify if does not break foreign dir and that reverse is also true. Also fixes a bug causing SEGVs during "lfs find [--mdt-count=[+,-], --mdt-hash=]" when handling a file (ie, "DIR *dir" is NULLL) in cb_find_init(). Signed-off-by: Bruno Faccini Change-Id: I3721b8f14578bf926a92da76375dae92dc8d764d Reviewed-on: https://review.whamcloud.com/34087 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Patrick Farrell Reviewed-by: Oleg Drokin --- lustre/doc/lfs-find.1 | 14 +- lustre/doc/lfs-setdirstripe.1 | 31 +++- lustre/include/lustre/lustreapi.h | 2 + lustre/include/lustre_lmv.h | 7 + lustre/include/obd.h | 5 +- lustre/include/uapi/linux/lustre/lustre_idl.h | 11 ++ lustre/include/uapi/linux/lustre/lustre_user.h | 31 +++- lustre/lfsck/lfsck_striped_dir.c | 45 ++++- lustre/llite/dir.c | 92 ++++++++-- lustre/llite/file.c | 5 + lustre/llite/llite_lib.c | 19 ++- lustre/lmv/lmv_intent.c | 14 ++ lustre/lmv/lmv_obd.c | 53 +++++- lustre/lod/lod_internal.h | 9 +- lustre/lod/lod_lov.c | 16 ++ lustre/lod/lod_object.c | 116 ++++++++++++- lustre/mdc/mdc_request.c | 15 +- lustre/mdd/mdd_dir.c | 3 +- lustre/mdt/mdt_lib.c | 14 ++ lustre/ptlrpc/pack_generic.c | 9 +- lustre/tests/Makefile.am | 2 + lustre/tests/create_foreign_dir.c | 71 ++++++++ lustre/tests/create_foreign_file.c | 2 +- lustre/tests/parse_foreign_dir.c | 100 +++++++++++ lustre/tests/sanity-lfsck.sh | 100 +++++++++++ lustre/tests/sanity.sh | 83 ++++++++- lustre/utils/lfs.c | 175 ++++++++++++++----- lustre/utils/liblustreapi.c | 226 +++++++++++++++++++++++-- lustre/utils/liblustreapi.map | 2 +- 29 files changed, 1166 insertions(+), 106 deletions(-) create mode 100644 lustre/tests/create_foreign_dir.c create mode 100644 lustre/tests/parse_foreign_dir.c diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 046d89a..f44a366 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -23,6 +23,7 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--name\fR|\fB-n <\fIpattern\fR>] [[\fB!\fR] \fB--ost\fR|\fB-O\fR <\fIindex\fR,...>] [[\fB!\fR] \fB--pool\fR <\fIpool\fR>] +[[\fB!\fR] \fB--foreign\fR [<\fItype\fR>]] [\fB--print\fR|\fB-P\fR] [\fB--print0\fR|\fB-0\fR] [[\fB!\fR] \fB--projid\fR |<\fIprojid\fR>] @@ -198,6 +199,15 @@ Layout was created with the specified .I pool name. For composite files, this may match the pool of any component. .TP +.BR --foreign[=] +Directory striping format is foreign (non-Lustre/free format) and is of type +.BR +if specified. Presently only +.BR none +or +.BR daos +are defined types , though 32-bit numeric types can also be used. +.TP .BR --print | -P Prints the file or directory name to standard output if it matches all specified parameters, one file per line with a trailing linefeed. @@ -298,7 +308,9 @@ Recursively list all mirrored files that have more than 2 mirrors. Recursively list all out-of-sync mirrored files. .TP .B $ lfs find ! --foreign=daos /mnt/lustre -Recursively list all but foreign files of daos type. +Recursively list all but foreign files/dirs of +.B daos +type. .SH BUGS The .B lfs find diff --git a/lustre/doc/lfs-setdirstripe.1 b/lustre/doc/lfs-setdirstripe.1 index 78cb3a2..9f52fa8 100644 --- a/lustre/doc/lfs-setdirstripe.1 +++ b/lustre/doc/lfs-setdirstripe.1 @@ -2,13 +2,18 @@ .SH NAME lfs setdirstripe, mkdir \- set striping pattern of a directory. .SH SYNOPSIS -.B lfs setdirstripe [\fR-cdDHioT\fR] \fIDIR\fR... +.B lfs setdirstripe [\fR-cdDHioTx\fR] \fIDIR\fR... .br .SH DESCRIPTION Create a striped directory with specified striping pattern. This lfs utility is similar to .BR lfs setstripe , but is used to create striped directory. +Can also be used to create directory with a foreign (free format) striping pattern (see +.BR --foreign +and +.BR --xattr +options). .B lfs mkdir is an alias for this command. .SH OPTIONS @@ -73,6 +78,23 @@ numeric .BR chmod (1). It is not affected by the current .BR umask (1p). +.TP +.BR \-\-foreign[=type] +Create a directory with a foreign (non-Lustre/free format, see +.BR \-\-xattr +option) striping. Where +.BR type +specifies a known foreign type (like +.BR none +, +.BR daos +, ...) or a 32-bit numeric type. +.TP +.BR \-\-flags =\fI\fR +Specify a bitmask of flags. +.TP +.BR \-x ", " \-\-xattr =\fISTRING\fR +Specify a string to be used as a foreign (free format) striping. .SH NOTE .PP The @@ -102,6 +124,13 @@ This creates a directory striped on two MDTs, whose first stripe is on .B MDT0001 (MDT index 1), and whose hash type is .BR all_char . +.TP +.B $ lfs mkdir --foreign=daos --xattr PUUID:CUUID /mnt/lustre/dir1 +This creates +.B dir1 +with foreign (non-lustre/free format) +.B PUUID:CUUID +striping/LMV EA. .SH AVAILABILITY The .B lfs setdirstripe diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 1f8dae2..074dd3d 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -341,6 +341,8 @@ int llapi_dir_set_default_lmv_stripe(const char *name, int stripe_offset, const char *pool_name); int llapi_dir_create(const char *name, mode_t mode, const struct llapi_stripe_param *param); +int llapi_dir_create_foreign(const char *name, mode_t mode, __u32 type, + __u32 flags, const char *value); int llapi_dir_create_pool(const char *name, int flags, int stripe_offset, int stripe_count, int stripe_pattern, const char *poolname); diff --git a/lustre/include/lustre_lmv.h b/lustre/include/lustre_lmv.h index d5fb751..3410b92 100644 --- a/lustre/include/lustre_lmv.h +++ b/lustre/include/lustre_lmv.h @@ -191,4 +191,11 @@ static inline bool lmv_is_known_hash_type(__u32 type) (type & LMV_HASH_TYPE_MASK) == LMV_HASH_TYPE_ALL_CHARS; } +static inline bool lmv_magic_supported(__u32 lum_magic) +{ + return lum_magic == LMV_USER_MAGIC || + lum_magic == LMV_USER_MAGIC_SPECIFIC || + lum_magic == LMV_MAGIC_FOREIGN; +} + #endif diff --git a/lustre/include/obd.h b/lustre/include/obd.h index ce41c32e..44c6ced 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -1037,7 +1037,10 @@ struct obd_ops { struct lustre_md { struct mdt_body *body; struct lu_buf layout; - struct lmv_stripe_md *lmv; + union { + struct lmv_stripe_md *lmv; + struct lmv_foreign_md *lfm; + }; #ifdef CONFIG_FS_POSIX_ACL struct posix_acl *posix_acl; #endif diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index e395e6c..bb9dd61 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -2202,11 +2202,21 @@ struct lmv_mds_md_v1 { struct lu_fid lmv_stripe_fids[0]; /* FIDs for each stripe */ }; +/* foreign LMV EA */ +struct lmv_foreign_md { + __u32 lfm_magic; /* magic number = LMV_MAGIC_FOREIGN */ + __u32 lfm_length; /* length of lfm_value */ + __u32 lfm_type; /* type, see LU_FOREIGN_TYPE_ */ + __u32 lfm_flags; /* flags, type specific */ + char lfm_value[]; /* free format value */ +}; + #define LMV_MAGIC_V1 0x0CD20CD0 /* normal stripe lmv magic */ #define LMV_MAGIC LMV_MAGIC_V1 /* #define LMV_USER_MAGIC 0x0CD30CD0 */ #define LMV_MAGIC_STRIPE 0x0CD40CD0 /* magic for dir sub_stripe */ +#define LMV_MAGIC_FOREIGN 0x0CD50CD0 /* magic for lmv foreign */ /* Right now only the lower part(0-16bits) of lmv_hash_type is being used, * and the higher part will be the flag to indicate the status of object, @@ -2262,6 +2272,7 @@ union lmv_mds_md { __u32 lmv_magic; struct lmv_mds_md_v1 lmv_md_v1; struct lmv_user_md lmv_user_md; + struct lmv_foreign_md lmv_foreign_md; }; static inline int lmv_mds_md_size(int stripe_count, unsigned int lmm_magic) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 3329aae..29e5260 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -630,7 +630,7 @@ struct lov_user_md_v3 { /* LOV EA user data (host-endian) */ struct lov_foreign_md { __u32 lfm_magic; /* magic number = LOV_MAGIC_FOREIGN */ __u32 lfm_length; /* length of lfm_value */ - __u32 lfm_type; /* type, see LOV_FOREIGN_TYPE_ */ + __u32 lfm_type; /* type, see LU_FOREIGN_TYPE_ */ __u32 lfm_flags; /* flags, type specific */ char lfm_value[]; }; @@ -799,19 +799,22 @@ enum lmv_hash_type { extern char *mdt_hash_name[LMV_HASH_TYPE_MAX]; -/** - * LOV foreign types - **/ -#define LOV_FOREIGN_TYPE_NONE 0 -#define LOV_FOREIGN_TYPE_DAOS 0xda05 -#define LOV_FOREIGN_TYPE_UNKNOWN UINT32_MAX - struct lustre_foreign_type { uint32_t lft_type; const char *lft_name; }; -extern struct lustre_foreign_type lov_foreign_type[]; +/** + * LOV/LMV foreign types + **/ +enum lustre_foreign_types { + LU_FOREIGN_TYPE_NONE = 0, + LU_FOREIGN_TYPE_DAOS = 0xda05, + /* must be the max/last one */ + LU_FOREIGN_TYPE_UNKNOWN = 0xffffffff, +}; + +extern struct lustre_foreign_type lu_foreign_types[]; /* Got this according to how get LOV_MAX_STRIPE_COUNT, see above, * (max buffer size - lmv+rpc header) / sizeof(struct lmv_user_mds_data) */ @@ -830,6 +833,16 @@ struct lmv_user_md_v1 { struct lmv_user_mds_data lum_objects[0]; } __attribute__((packed)); +static inline __u32 lmv_foreign_to_md_stripes(__u32 size) +{ + if (size <= sizeof(struct lmv_user_md)) + return 0; + + size -= sizeof(struct lmv_user_md); + return (size + sizeof(struct lmv_user_mds_data) - 1) / + sizeof(struct lmv_user_mds_data); +} + static inline int lmv_user_md_size(int stripes, int lmm_magic) { int size = sizeof(struct lmv_user_md); diff --git a/lustre/lfsck/lfsck_striped_dir.c b/lustre/lfsck/lfsck_striped_dir.c index 33c57d7..843381c 100644 --- a/lustre/lfsck/lfsck_striped_dir.c +++ b/lustre/lfsck/lfsck_striped_dir.c @@ -842,6 +842,7 @@ int lfsck_read_stripe_lmv(const struct lu_env *env, struct dt_object *obj, struct lfsck_thread_info *info = lfsck_env_info(env); struct lu_buf *buf = &info->lti_buf; int size = sizeof(*lmv) + sizeof(struct lu_fid) * 2; + struct lmv_foreign_md *lfm; int rc; dt_read_lock(env, obj, 0); @@ -850,17 +851,49 @@ int lfsck_read_stripe_lmv(const struct lu_env *env, struct dt_object *obj, rc = dt_xattr_get(env, obj, buf, XATTR_NAME_LMV); if (unlikely(rc == -ERANGE)) { buf = &info->lti_big_buf; - lu_buf_check_and_alloc(buf, size); - rc = dt_xattr_get(env, obj, buf, XATTR_NAME_LMV); /* For the in-migration directory, its LMV EA contains * not only the LMV header, but also the FIDs for both - * source and target. So the LMV EA size is larger. */ - if (rc == size) { - rc = sizeof(*lmv); - memcpy(lmv, buf->lb_buf, rc); + * source and target. So the LMV EA size is larger. + * Or may be this is a foreign LMV */ + rc = dt_xattr_get(env, obj, &LU_BUF_NULL, XATTR_NAME_LMV); + if (rc > sizeof(*lmv)) { + int rc1; + + lu_buf_check_and_alloc(buf, rc); + rc1 = dt_xattr_get(env, obj, buf, XATTR_NAME_LMV); + if (rc != rc1) + rc = -EINVAL; + } else { + rc = -EINVAL; } } dt_read_unlock(env, obj); + + if (rc > 0 && rc > offsetof(typeof(*lfm), lfm_value) && + *((__u32 *)buf->lb_buf) == LMV_MAGIC_FOREIGN) { + __u32 value_len; + + lfm = buf->lb_buf; + value_len = le32_to_cpu(lfm->lfm_length); + CDEBUG(D_INFO, + "foreign LMV EA, magic %x, len %u, type %x, flags %x, for dir "DFID"\n", + le32_to_cpu(lfm->lfm_magic), value_len, + le32_to_cpu(lfm->lfm_type), le32_to_cpu(lfm->lfm_flags), + PFID(lfsck_dto2fid(obj))); + + if (rc != value_len + offsetof(typeof(*lfm), lfm_value)) + CDEBUG(D_LFSCK, + "foreign LMV EA internal size %u does not match EA full size %d for dir "DFID"\n", + value_len, rc, PFID(lfsck_dto2fid(obj))); + + /* no further usage/decode of foreign LMV outside */ + return -ENODATA; + } + + if (rc == size) { + rc = sizeof(*lmv); + memcpy(lmv, buf->lb_buf, rc); + } if (rc != sizeof(*lmv)) return rc > 0 ? -EINVAL : rc; diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index d04196e..9fff515 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -367,6 +367,14 @@ static int ll_readdir(struct file *filp, void *cookie, filldir_t filldir) LUSTRE_OPC_ANY, inode); if (IS_ERR(op_data)) GOTO(out, rc = PTR_ERR(op_data)); + + /* foreign dirs are browsed out of Lustre */ + if (unlikely(op_data->op_mea1 != NULL && + op_data->op_mea1->lsm_md_magic == LMV_MAGIC_FOREIGN)) { + ll_finish_md_op_data(op_data); + RETURN(-ENODATA); + } + op_data->op_fid3 = pfid; #ifdef HAVE_DIR_CONTEXT @@ -454,14 +462,22 @@ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, int err; ENTRY; - if (unlikely(lump->lum_magic != LMV_USER_MAGIC && - lump->lum_magic != LMV_USER_MAGIC_SPECIFIC)) + if (unlikely(!lmv_magic_supported(lump->lum_magic))) RETURN(-EINVAL); - CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) name %s " - "stripe_offset %d, stripe_count: %u\n", - PFID(ll_inode2fid(parent)), parent, dirname, - (int)lump->lum_stripe_offset, lump->lum_stripe_count); + if (lump->lum_magic != LMV_MAGIC_FOREIGN) { + CDEBUG(D_VFSTRACE, + "VFS Op:inode="DFID"(%p) name %s stripe_offset %d, stripe_count: %u\n", + PFID(ll_inode2fid(parent)), parent, dirname, + (int)lump->lum_stripe_offset, lump->lum_stripe_count); + } else { + struct lmv_foreign_md *lfm = (struct lmv_foreign_md *)lump; + + CDEBUG(D_VFSTRACE, + "VFS Op:inode="DFID"(%p) name %s foreign, length %u, value '%.*s'\n", + PFID(ll_inode2fid(parent)), parent, dirname, + lfm->lfm_length, lfm->lfm_length, lfm->lfm_value); + } if (lump->lum_stripe_count > 1 && !(exp_connect_flags(sbi->ll_md_exp) & OBD_CONNECT_DIR_STRIPE)) @@ -471,8 +487,7 @@ static int ll_dir_setdirstripe(struct dentry *dparent, struct lmv_user_md *lump, !OBD_FAIL_CHECK(OBD_FAIL_LLITE_NO_CHECK_DEAD)) RETURN(-ENOENT); - if (lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC) && - lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC_SPECIFIC)) + if (unlikely(!lmv_magic_supported(cpu_to_le32(lump->lum_magic)))) lustre_swab_lmv_user_md(lump); if (!IS_POSIXACL(parent) || !exp_connect_umask(ll_i2mdexp(parent))) @@ -768,6 +783,17 @@ int ll_dir_getstripe(struct inode *inode, void **plmm, int *plmm_size, } } break; + case LMV_MAGIC_FOREIGN: { + struct lmv_foreign_md *lfm = (struct lmv_foreign_md *)lmm; + + if (LMV_MAGIC_FOREIGN != cpu_to_le32(LMV_MAGIC_FOREIGN)) { + __swab32s(&lfm->lfm_magic); + __swab32s(&lfm->lfm_length); + __swab32s(&lfm->lfm_type); + __swab32s(&lfm->lfm_flags); + } + break; + } default: CERROR("unknown magic: %lX\n", (unsigned long)lmm->lmm_magic); rc = -EPROTO; @@ -1361,9 +1387,22 @@ out_free: lum = (struct lmv_user_md *)data->ioc_inlbuf2; lumlen = data->ioc_inllen2; - if ((lum->lum_magic != LMV_USER_MAGIC && - lum->lum_magic != LMV_USER_MAGIC_SPECIFIC) || + if (!lmv_magic_supported(lum->lum_magic)) { + CERROR("%s: wrong lum magic %x : rc = %d\n", filename, + lum->lum_magic, -EINVAL); + GOTO(lmv_out_free, rc = -EINVAL); + } + + if ((lum->lum_magic == LMV_USER_MAGIC || + lum->lum_magic == LMV_USER_MAGIC_SPECIFIC) && lumlen < sizeof(*lum)) { + CERROR("%s: wrong lum size %d for magic %x : rc = %d\n", + filename, lumlen, lum->lum_magic, -EINVAL); + GOTO(lmv_out_free, rc = -EINVAL); + } + + if (lum->lum_magic == LMV_MAGIC_FOREIGN && + lumlen < sizeof(struct lmv_foreign_md)) { CERROR("%s: wrong lum magic %x or size %d: rc = %d\n", filename, lum->lum_magic, lumlen, -EFAULT); GOTO(lmv_out_free, rc = -EINVAL); @@ -1491,7 +1530,25 @@ out: GOTO(finish_req, rc); } - stripe_count = lmv_mds_md_stripe_count_get(lmm); + /* if foreign LMV case, fake stripes number */ + if (lmm->lmv_magic == LMV_MAGIC_FOREIGN) { + struct lmv_foreign_md *lfm; + + lfm = (struct lmv_foreign_md *)lmm; + if (lfm->lfm_length < XATTR_SIZE_MAX - + offsetof(typeof(*lfm), lfm_value)) { + __u32 size = lfm->lfm_length + + offsetof(typeof(*lfm), lfm_value); + + stripe_count = lmv_foreign_to_md_stripes(size); + } else { + CERROR("invalid %d foreign size returned\n", + lfm->lfm_length); + return -EINVAL; + } + } else { + stripe_count = lmv_mds_md_stripe_count_get(lmm); + } if (max_stripe_count < stripe_count) { lum.lum_stripe_count = stripe_count; if (copy_to_user(ulmv, &lum, sizeof(lum))) @@ -1499,6 +1556,19 @@ out: GOTO(finish_req, rc = -E2BIG); } + /* enough room on user side and foreign case */ + if (lmm->lmv_magic == LMV_MAGIC_FOREIGN) { + struct lmv_foreign_md *lfm; + __u32 size; + + lfm = (struct lmv_foreign_md *)lmm; + size = lfm->lfm_length + + offsetof(struct lmv_foreign_md, lfm_value); + if (copy_to_user(ulmv, lfm, size)) + GOTO(finish_req, rc = -EFAULT); + GOTO(finish_req, rc); + } + lum_size = lmv_user_md_size(stripe_count, LMV_USER_MAGIC_SPECIFIC); OBD_ALLOC(tmp, lum_size); diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 8d2368b..89c354d 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -4474,6 +4474,11 @@ static int ll_merge_md_attr(struct inode *inode) int rc; LASSERT(lli->lli_lsm_md != NULL); + + /* foreign dir is not striped dir */ + if (lli->lli_lsm_md->lsm_md_magic == LMV_MAGIC_FOREIGN) + RETURN(0); + down_read(&lli->lli_lsm_sem); rc = md_merge_attr(ll_i2mdexp(inode), ll_i2info(inode)->lli_lsm_md, &attr, ll_md_blocking_ast); diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 273af31..da0db39 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1412,8 +1412,12 @@ static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md) /* * if dir layout mismatch, check whether version is increased, which * means layout is changed, this happens in dir migration and lfsck. + * + * foreign LMV should not change. */ - if (lli->lli_lsm_md && !lsm_md_eq(lli->lli_lsm_md, lsm)) { + if (lli->lli_lsm_md && + lli->lli_lsm_md->lsm_md_magic != LMV_MAGIC_FOREIGN && + !lsm_md_eq(lli->lli_lsm_md, lsm)) { if (lsm->lsm_md_layout_version <= lli->lli_lsm_md->lsm_md_layout_version) { CERROR("%s: "DFID" dir layout mismatch:\n", @@ -1434,6 +1438,15 @@ static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md) if (!lli->lli_lsm_md) { struct cl_attr *attr; + if (lsm->lsm_md_magic == LMV_MAGIC_FOREIGN) { + /* set md->lmv to NULL, so the following free lustre_md + * will not free this lsm */ + md->lmv = NULL; + lli->lli_lsm_md = lsm; + up_write(&lli->lli_lsm_sem); + RETURN(0); + } + rc = ll_init_lsm_md(inode, md); up_write(&lli->lli_lsm_sem); if (rc != 0) @@ -2396,7 +2409,7 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req, rc = md_get_lustre_md(sbi->ll_md_exp, req, sbi->ll_dt_exp, sbi->ll_md_exp, &md); if (rc != 0) - GOTO(cleanup, rc); + GOTO(out, rc); if (*inode) { rc = ll_update_inode(*inode, &md); @@ -2464,9 +2477,9 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req, GOTO(out, rc = 0); out: + /* cleanup will be done if necessary */ md_free_lustre_md(sbi->ll_md_exp, &md); -cleanup: if (rc != 0 && it != NULL && it->it_op & IT_OPEN) ll_open_cleanup(sb != NULL ? sb : (*inode)->i_sb, req); diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c index 7cac3dc..bb94c26 100644 --- a/lustre/lmv/lmv_intent.c +++ b/lustre/lmv/lmv_intent.c @@ -276,6 +276,11 @@ static int lmv_intent_open(struct obd_export *exp, struct md_op_data *op_data, ENTRY; + /* do not allow file creation in foreign dir */ + if ((it->it_op & IT_CREAT) && op_data->op_mea1 != NULL && + op_data->op_mea1->lsm_md_magic == LMV_MAGIC_FOREIGN) + RETURN(-ENODATA); + if ((it->it_op & IT_CREAT) && !(flags & MDS_OPEN_BY_FID)) { /* don't allow create under dir with bad hash */ if (lmv_is_dir_bad_hash(op_data->op_mea1)) @@ -422,6 +427,15 @@ lmv_intent_lookup(struct obd_export *exp, struct md_op_data *op_data, int rc; ENTRY; + /* foreign dir is not striped */ + if (op_data->op_mea1 && + op_data->op_mea1->lsm_md_magic == LMV_MAGIC_FOREIGN) { + /* only allow getattr/lookup for itself */ + if (op_data->op_name != NULL) + RETURN(-ENODATA); + RETURN(0); + } + retry: tgt = lmv_locate_tgt(lmv, op_data, &op_data->op_fid1); if (IS_ERR(tgt)) diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 2df8aa3..be2691e 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -1176,15 +1176,23 @@ static int lmv_placement_policy(struct obd_device *obd, * 1. See if the stripe offset is specified by lum. * 2. Then check if there is default stripe offset. * 3. Finally choose MDS by name hash if the parent - * is striped directory. (see lmv_locate_tgt()). */ + * is striped directory. (see lmv_locate_tgt()). + * + * presently explicit MDT location is not supported + * for foreign dirs (as it can't be embedded into free + * format LMV, like with lum_stripe_offset), so we only + * rely on default stripe offset or then name hashing. + */ if (op_data->op_cli_flags & CLI_SET_MEA && lum != NULL && + le32_to_cpu(lum->lum_magic != LMV_MAGIC_FOREIGN) && le32_to_cpu(lum->lum_stripe_offset) != (__u32)-1) { *mds = le32_to_cpu(lum->lum_stripe_offset); } else if (op_data->op_default_stripe_offset != (__u32)-1) { *mds = op_data->op_default_stripe_offset; op_data->op_mds = *mds; /* Correct the stripe offset in lum */ - if (lum != NULL) + if (lum != NULL && + le32_to_cpu(lum->lum_magic != LMV_MAGIC_FOREIGN)) lum->lum_stripe_offset = cpu_to_le32(*mds); } else { *mds = op_data->op_mds; @@ -1633,6 +1641,10 @@ lmv_locate_tgt(struct lmv_obd *lmv, struct md_op_data *op_data, struct lmv_oinfo *oinfo; struct lmv_tgt_desc *tgt; + /* foreign dir is not striped dir */ + if (lsm && lsm->lsm_md_magic == LMV_MAGIC_FOREIGN) + return ERR_PTR(-ENODATA); + /* During creating VOLATILE file, it should honor the mdt * index if the file under striped dir is being restored, see * ct_restore(). */ @@ -2696,6 +2708,10 @@ int lmv_read_page(struct obd_export *exp, struct md_op_data *op_data, ENTRY; if (unlikely(lsm != NULL)) { + /* foreign dir is not striped dir */ + if (lsm->lsm_md_magic == LMV_MAGIC_FOREIGN) + return -ENODATA; + rc = lmv_striped_read_page(exp, op_data, cb_op, offset, ppage); RETURN(rc); } @@ -3011,6 +3027,16 @@ static int lmv_unpackmd(struct obd_export *exp, struct lmv_stripe_md **lsmp, /* Free memmd */ if (lsm != NULL && lmm == NULL) { int i; + struct lmv_foreign_md *lfm = (struct lmv_foreign_md *)lsm; + + if (lfm->lfm_magic == LMV_MAGIC_FOREIGN) { + size_t lfm_size; + + lfm_size = lfm->lfm_length + offsetof(typeof(*lfm), + lfm_value[0]); + OBD_FREE_LARGE(lfm, lfm_size); + RETURN(0); + } for (i = 0; i < lsm->lsm_md_stripe_count; i++) iput(lsm->lsm_md_oinfo[i].lmo_root); @@ -3020,6 +3046,25 @@ static int lmv_unpackmd(struct obd_export *exp, struct lmv_stripe_md **lsmp, RETURN(0); } + /* foreign lmv case */ + if (le32_to_cpu(lmm->lmv_magic) == LMV_MAGIC_FOREIGN) { + struct lmv_foreign_md *lfm = (struct lmv_foreign_md *)lsm; + + if (lfm == NULL) { + OBD_ALLOC_LARGE(lfm, lmm_size); + if (lfm == NULL) + RETURN(-ENOMEM); + *lsmp = (struct lmv_stripe_md *)lfm; + } + lfm->lfm_magic = le32_to_cpu(lmm->lmv_foreign_md.lfm_magic); + lfm->lfm_length = le32_to_cpu(lmm->lmv_foreign_md.lfm_length); + lfm->lfm_type = le32_to_cpu(lmm->lmv_foreign_md.lfm_type); + lfm->lfm_flags = le32_to_cpu(lmm->lmv_foreign_md.lfm_flags); + memcpy(&lfm->lfm_value, &lmm->lmv_foreign_md.lfm_value, + lfm->lfm_length); + RETURN(lmm_size); + } + if (le32_to_cpu(lmm->lmv_magic) == LMV_MAGIC_STRIPE) RETURN(-EPERM); @@ -3344,6 +3389,10 @@ static int lmv_merge_attr(struct obd_export *exp, int rc; int i; + /* foreign dir is not striped dir */ + if (lsm->lsm_md_magic == LMV_MAGIC_FOREIGN) + return 0; + rc = lmv_revalidate_slaves(exp, lsm, cb_blocking, 0); if (rc < 0) return rc; diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 4304639..45df47d 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -323,7 +323,9 @@ struct lod_object { __u32 ldo_dir_slave_stripe:1, ldo_dir_striped:1, /* the stripe has been loaded */ - ldo_dir_stripe_loaded:1; + ldo_dir_stripe_loaded:1, + /* foreign directory */ + ldo_dir_is_foreign; /* * default striping is not cached, so this field is * invalid after create, make sure it's used by @@ -339,6 +341,11 @@ struct lod_object { size_t ldo_foreign_lov_size; }; struct { + /* foreign/raw format LMV */ + char *ldo_foreign_lmv; + size_t ldo_foreign_lmv_size; + }; + struct { /* file stripe (LOV) */ struct lod_layout_component *ldo_comp_entries; /* slave stripes of striped directory (LMV) */ diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 2197e41..3223e98 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -1513,6 +1513,22 @@ int lod_striping_load(const struct lu_env *env, struct lod_object *lo) lo->ldo_comp_cached = 1; } else if (S_ISDIR(lod2lu_obj(lo)->lo_header->loh_attr)) { rc = lod_get_lmv_ea(env, lo); + if (rc > sizeof(struct lmv_foreign_md)) { + struct lmv_foreign_md *lfm = info->lti_ea_store; + + if (le32_to_cpu(lfm->lfm_magic) == LMV_MAGIC_FOREIGN) { + lo->ldo_foreign_lmv = info->lti_ea_store; + lo->ldo_foreign_lmv_size = + info->lti_ea_store_size; + info->lti_ea_store = NULL; + info->lti_ea_store_size = 0; + + lo->ldo_dir_stripe_loaded = 1; + lo->ldo_dir_is_foreign = 1; + GOTO(unlock, rc = 0); + } + } + if (rc < (typeof(rc))sizeof(struct lmv_mds_md_v1)) { /* Let's set stripe_loaded to avoid further * stripe loading especially for non-stripe directory, diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 9bd60af..7224f4c 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -1454,21 +1454,35 @@ static int lod_xattr_get(const struct lu_env *env, struct dt_object *dt, rc = dt_xattr_get(env, dt_object_child(dt), buf, name); if (strcmp(name, XATTR_NAME_LMV) == 0) { struct lmv_mds_md_v1 *lmv1; + struct lmv_foreign_md *lfm; int rc1 = 0; if (rc > (typeof(rc))sizeof(*lmv1)) RETURN(rc); - if (rc < (typeof(rc))sizeof(*lmv1)) + /* short (<= sizeof(struct lmv_mds_md_v1)) foreign LMV case */ + /* XXX empty foreign LMV is not allowed */ + if (rc <= offsetof(typeof(*lfm), lfm_value)) RETURN(rc = rc > 0 ? -EINVAL : rc); if (buf->lb_buf == NULL || buf->lb_len == 0) { CLASSERT(sizeof(*lmv1) <= sizeof(info->lti_key)); + /* lti_buf is large enough for *lmv1 or a short + * (<= sizeof(struct lmv_mds_md_v1)) foreign LMV + */ info->lti_buf.lb_buf = info->lti_key; info->lti_buf.lb_len = sizeof(*lmv1); rc = dt_xattr_get(env, dt_object_child(dt), &info->lti_buf, name); + if (unlikely(rc <= offsetof(typeof(*lfm), + lfm_value))) + RETURN(rc = rc > 0 ? -EINVAL : rc); + + lfm = info->lti_buf.lb_buf; + if (le32_to_cpu(lfm->lfm_magic) == LMV_MAGIC_FOREIGN) + RETURN(rc); + if (unlikely(rc != sizeof(*lmv1))) RETURN(rc = rc > 0 ? -EINVAL : rc); @@ -1481,6 +1495,13 @@ static int lod_xattr_get(const struct lu_env *env, struct dt_object *dt, le32_to_cpu(lmv1->lmv_stripe_count), LMV_MAGIC_V1); } else { + lfm = buf->lb_buf; + if (le32_to_cpu(lfm->lfm_magic) == LMV_MAGIC_FOREIGN) + RETURN(rc); + + if (rc != sizeof(*lmv1)) + RETURN(rc = rc > 0 ? -EINVAL : rc); + rc1 = lod_load_lmv_shards(env, lod_dt_obj(dt), buf, false); } @@ -1662,6 +1683,10 @@ int lod_parse_dir_striping(const struct lu_env *env, struct lod_object *lo, LASSERT(mutex_is_locked(&lo->ldo_layout_mutex)); + /* XXX may be useless as not called for foreign LMV ?? */ + if (le32_to_cpu(lmv1->lmv_magic) == LMV_MAGIC_FOREIGN) + RETURN(0); + if (le32_to_cpu(lmv1->lmv_magic) == LMV_MAGIC_STRIPE) { lo->ldo_dir_slave_stripe = 1; RETURN(0); @@ -2050,6 +2075,27 @@ out_free: } /** + * + * Alloc cached foreign LMV + * + * \param[in] lo object + * \param[in] size size of foreign LMV + * + * \retval 0 on success + * \retval negative if failed + */ +int lod_alloc_foreign_lmv(struct lod_object *lo, size_t size) +{ + OBD_ALLOC_LARGE(lo->ldo_foreign_lmv, size); + if (lo->ldo_foreign_lmv == NULL) + return -ENOMEM; + lo->ldo_foreign_lmv_size = size; + lo->ldo_dir_is_foreign = 1; + + return 0; +} + +/** * Declare create striped md object. * * The function declares intention to create a striped directory. This is a @@ -2086,8 +2132,16 @@ static int lod_declare_xattr_set_lmv(const struct lu_env *env, le32_to_cpu(lum->lum_magic), le32_to_cpu(lum->lum_stripe_count), (int)le32_to_cpu(lum->lum_stripe_offset)); - if (lo->ldo_dir_stripe_count == 0) + if (lo->ldo_dir_stripe_count == 0) { + if (lo->ldo_dir_is_foreign) { + rc = lod_alloc_foreign_lmv(lo, lum_buf->lb_len); + if (rc != 0) + GOTO(out, rc); + memcpy(lo->ldo_foreign_lmv, lum, lum_buf->lb_len); + lo->ldo_dir_stripe_loaded = 1; + } GOTO(out, rc = 0); + } /* prepare dir striped objects */ rc = lod_prep_md_striped_create(env, dt, attr, lum, dof, th); @@ -3629,8 +3683,15 @@ static int lod_xattr_set_lmv(const struct lu_env *env, struct dt_object *dt, /* The stripes are supposed to be allocated in declare phase, * if there are no stripes being allocated, it will skip */ - if (lo->ldo_dir_stripe_count == 0) + if (lo->ldo_dir_stripe_count == 0) { + if (lo->ldo_dir_is_foreign) { + rc = lod_sub_xattr_set(env, dt_object_child(dt), buf, + XATTR_NAME_LMV, fl, th); + if (rc != 0) + RETURN(rc); + } RETURN(0); + } rc = dt_attr_get(env, dt_object_child(dt), attr); if (rc != 0) @@ -3822,6 +3883,26 @@ static int lod_dir_striping_create_internal(const struct lu_env *env, th); if (rc != 0) RETURN(rc); + } else { + /* foreign LMV EA case */ + if (lmu) { + struct lmv_foreign_md *lfm = lmu->lb_buf; + + if (lfm->lfm_magic == LMV_MAGIC_FOREIGN) { + rc = lod_declare_xattr_set_lmv(env, dt, attr, + lmu, dof, th); + } + } else { + if (lo->ldo_dir_is_foreign) { + LASSERT(lo->ldo_foreign_lmv != NULL && + lo->ldo_foreign_lmv_size > 0); + info->lti_buf.lb_buf = lo->ldo_foreign_lmv; + info->lti_buf.lb_len = lo->ldo_foreign_lmv_size; + lmu = &info->lti_buf; + rc = lod_xattr_set_lmv(env, dt, lmu, + XATTR_NAME_LMV, 0, th); + } + } } /* Transfer default LMV striping from the parent */ @@ -4765,6 +4846,17 @@ static void lod_ah_init(const struct lu_env *env, /* other default values are 0 */ lc->ldo_dir_stripe_offset = -1; + /* no default striping configuration is needed for + * foreign dirs + */ + if (ah->dah_eadata != NULL && ah->dah_eadata_len != 0 && + le32_to_cpu(lum1->lum_magic) == LMV_MAGIC_FOREIGN) { + lc->ldo_dir_is_foreign = true; + /* keep stripe_count 0 and stripe_offset -1 */ + CDEBUG(D_INFO, "no default striping for foreign dir\n"); + RETURN_EXIT; + } + /* * If parent object is not root directory, * then get default striping from parent object. @@ -6858,6 +6950,21 @@ void lod_free_foreign_lov(struct lod_object *lo) /** * + * Free cached foreign LMV + * + * \param[in] lo object + */ +void lod_free_foreign_lmv(struct lod_object *lo) +{ + if (lo->ldo_foreign_lmv != NULL) + OBD_FREE_LARGE(lo->ldo_foreign_lmv, lo->ldo_foreign_lmv_size); + lo->ldo_foreign_lmv = NULL; + lo->ldo_foreign_lmv_size = 0; + lo->ldo_dir_is_foreign = 0; +} + +/** + * * Release resources associated with striping. * * If the object is striped (regular or directory), then release @@ -6874,6 +6981,9 @@ void lod_striping_free_nolock(const struct lu_env *env, struct lod_object *lo) if (unlikely(lo->ldo_is_foreign)) { lod_free_foreign_lov(lo); lo->ldo_comp_cached = 0; + } else if (unlikely(lo->ldo_dir_is_foreign)) { + lod_free_foreign_lmv(lo); + lo->ldo_dir_stripe_loaded = 0; } else if (lo->ldo_stripe != NULL) { LASSERT(lo->ldo_comp_entries == NULL); LASSERT(lo->ldo_dir_stripes_allocated > 0); diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 048c25a..0db1628 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -602,10 +602,17 @@ int mdc_get_lustre_md(struct obd_export *exp, struct ptlrpc_request *req, GOTO(out, rc); if (rc < (typeof(rc))sizeof(*md->lmv)) { - CDEBUG(D_INFO, "size too small: " - "rc < sizeof(*md->lmv) (%d < %d)\n", - rc, (int)sizeof(*md->lmv)); - GOTO(out, rc = -EPROTO); + struct lmv_foreign_md *lfm = md->lfm; + + /* short (< sizeof(struct lmv_stripe_md)) + * foreign LMV case + */ + if (lfm->lfm_magic != LMV_MAGIC_FOREIGN) { + CDEBUG(D_INFO, + "size too small: rc < sizeof(*md->lmv) (%d < %d)\n", + rc, (int)sizeof(*md->lmv)); + GOTO(out, rc = -EPROTO); + } } } } diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index 0072c4a..40afae5 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -2020,8 +2020,7 @@ static int mdd_create_sanity_check(const struct lu_env *env, spec->u.sp_ea.eadata != NULL && spec->u.sp_ea.eadatalen > 0) { const struct lmv_user_md *lum = spec->u.sp_ea.eadata; - if (le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC && - le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC_SPECIFIC && + if (!lmv_magic_supported(le32_to_cpu(lum->lum_magic)) && le32_to_cpu(lum->lum_magic) != LMV_USER_MAGIC_V0) { rc = -EINVAL; CERROR("%s: invalid lmv_user_md: magic = %x, " diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index 4cda802..b6619fd 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -681,11 +681,25 @@ void mdt_dump_lmm(int level, const struct lov_mds_md *lmm, __u64 valid) void mdt_dump_lmv(unsigned int level, const union lmv_mds_md *lmv) { const struct lmv_mds_md_v1 *lmm1; + const struct lmv_foreign_md *lfm; int i; if (likely(!cfs_cdebug_show(level, DEBUG_SUBSYSTEM))) return; + /* foreign LMV case */ + lfm = &lmv->lmv_foreign_md; + if (le32_to_cpu(lfm->lfm_magic) == LMV_MAGIC_FOREIGN) { + CDEBUG_LIMIT(level, + "foreign magic 0x%08X, length %u, type %u, flags %u, value '%.*s'\n", + le32_to_cpu(lfm->lfm_magic), + le32_to_cpu(lfm->lfm_length), + le32_to_cpu(lfm->lfm_type), + le32_to_cpu(lfm->lfm_flags), + le32_to_cpu(lfm->lfm_length), lfm->lfm_value); + return; + } + lmm1 = &lmv->lmv_md_v1; CDEBUG(level, "magic 0x%08X, master %#X stripe_count %#x hash_type %#x\n", diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 1493726..b5ec17f 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2136,8 +2136,15 @@ EXPORT_SYMBOL(lustre_swab_lmv_user_md_objects); void lustre_swab_lmv_user_md(struct lmv_user_md *lum) { - __u32 count = lum->lum_stripe_count; + __u32 count; + if (lum->lum_magic == LMV_MAGIC_FOREIGN) { + __swab32s(&lum->lum_magic); + __swab32s(&((struct lmv_foreign_md *)lum)->lfm_length); + return; + } + + count = lum->lum_stripe_count; __swab32s(&lum->lum_magic); __swab32s(&lum->lum_stripe_count); __swab32s(&lum->lum_stripe_offset); diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 5d4b23e..68fd6ae 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -76,6 +76,7 @@ THETESTS += llapi_layout_test orphan_linkea_check llapi_hsm_test THETESTS += group_lock_test llapi_fid_test sendfile_grouplock mmap_cat THETESTS += swap_lock_test lockahead_test mirror_io mmap_mknod_test THETESTS += create_foreign_file parse_foreign_file +THETESTS += create_foreign_dir parse_foreign_dir if TESTS if MPITESTS @@ -107,4 +108,5 @@ lockahead_test_LDADD = $(LIBLUSTREAPI) mirror_io_LDADD = $(LIBLUSTREAPI) ll_dirstripe_verify_LDADD = $(LIBLUSTREAPI) flocks_test_LDADD = $(LIBLUSTREAPI) $(PTHREAD_LIBS) +create_foreign_dir_LDADD = $(LIBLUSTREAPI) endif # TESTS diff --git a/lustre/tests/create_foreign_dir.c b/lustre/tests/create_foreign_dir.c new file mode 100644 index 0000000..9fa508c --- /dev/null +++ b/lustre/tests/create_foreign_dir.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char **argv) +{ + char *dir = "foreign_dir", *end; + char *xval = "UUID@UUID"; + mode_t mode = 0700; + __u32 type = LU_FOREIGN_TYPE_DAOS, flags = 0; + int c, rc; + + while ((c = getopt(argc, argv, "hd:f:m:t:x:")) != -1) { + switch (c) { + case 'd': + dir = optarg; + break; + case 'x': + xval = optarg; + break; + case 'm': + mode = strtoul(optarg, &end, 8); + if (*end != '\0') { + fprintf(stderr, + "%s: invalid mode '%s'\n", argv[0], + optarg); + exit(1); + } + break; + case 'f': + flags = strtoul(optarg, &end, 0); + if (*end != '\0') { + fprintf(stderr, + "%s: invalid flags '%s'\n", argv[0], + optarg); + exit(1); + } + break; + case 't': + type = strtoul(optarg, &end, 0); + if (*end != '\0') { + fprintf(stderr, + "%s: invalid type '%s'\n", argv[0], + optarg); + exit(1); + } + break; + case 'h': + default: + fprintf(stderr, + "Usage: %s [-d ] [-m ] [-x ] [-t ] [-f ]\n", + argv[0]); + exit(0); + break; + } + } + + rc = llapi_dir_create_foreign(dir, mode, type, flags, xval); + if (rc < 0) + fprintf(stderr, "llapi_dir_create_foreign() error : %d\n", rc); + + return rc; +} diff --git a/lustre/tests/create_foreign_file.c b/lustre/tests/create_foreign_file.c index e270135..ea29ceb 100644 --- a/lustre/tests/create_foreign_file.c +++ b/lustre/tests/create_foreign_file.c @@ -17,7 +17,7 @@ int main(int argc, char **argv) size_t len; struct lov_foreign_md *lfm; char *end; - __u32 type = LOV_FOREIGN_TYPE_DAOS, flags = 0xda08; + __u32 type = LU_FOREIGN_TYPE_DAOS, flags = 0xda08; while ((c = getopt(argc, argv, "f:x:t:F:")) != -1) { switch (c) { diff --git a/lustre/tests/parse_foreign_dir.c b/lustre/tests/parse_foreign_dir.c new file mode 100644 index 0000000..a0499bd --- /dev/null +++ b/lustre/tests/parse_foreign_dir.c @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char **argv) +{ + int c, i; + char *dname = "DIR"; + size_t len, len2; + struct lmv_foreign_md *lfm; + + while ((c = getopt(argc, argv, "d:")) != -1) { + switch (c) { + case 'd': + dname = optarg; + break; + case 'h': + fprintf(stderr, "Usage: %s -d \n", argv[0]); + break; + } + } + + len = getxattr(dname, "trusted.lmv", NULL, 0); + if (len == -1) { + perror("getxattr()"); + exit(1); + } + if (len > XATTR_SIZE_MAX || len <= 0) { + fprintf(stderr, "invalid LMV EA length %zu " + "(must be 0 < len < " + "XATTR_SIZE_MAX(%u))\n", len, + XATTR_SIZE_MAX); + exit(1); + } + + lfm = malloc(len); + if (lfm == NULL) { + perror("malloc()"); + exit(1); + } + + len2 = getxattr(dname, "trusted.lmv", lfm, len); + if (len2 == -1) { + perror("getxattr()"); + exit(1); + } + + if (len != len2) + fprintf(stderr, "trusted.lmv xattr size changed, before=%zu " + "now=%zu\n", len, len2); + + if (len2 < offsetof(struct lmv_foreign_md, lfm_value)) { + fprintf(stderr, "trusted.lov size=%zu too small\n", len2); + fprintf(stderr, "printing its content in hex anyway :\n"); + for (i = 0; i < len2; i++) + fprintf(stderr, "%02x", *((char *)lfm + i)); + exit(1); + } + + + if (lfm->lfm_magic != LMV_MAGIC_FOREIGN) { + if (lfm->lfm_magic == bswap_32(LMV_MAGIC_FOREIGN)) + fprintf(stderr, "magic is swapped\n"); + else + fprintf(stderr, "wrong magic=(0x%x)\n", lfm->lfm_magic); + } + + if (lfm->lfm_length != len2 - offsetof(typeof(*lfm), lfm_value)) { + if (bswap_32(lfm->lfm_length) == len2 - offsetof(typeof(*lfm), + lfm_value)) + fprintf(stderr, "length is swapped\n"); + else + fprintf(stderr, "wrong internal length=%u(0x%x) vs " + "xattr size=%zu\n", lfm->lfm_length, + lfm->lfm_length, len2); + } + + fprintf(stdout, "lmv_xattr_size: %zu\n", len2); + fprintf(stdout, "lmv_foreign_magic: 0x%x\n", lfm->lfm_magic); + fprintf(stdout, "lmv_foreign_size: %u\n", lfm->lfm_length); + fprintf(stdout, "lmv_foreign_type: %u\n", lfm->lfm_type); + fprintf(stdout, "lmv_foreign_flags: %u\n", lfm->lfm_flags); + fprintf(stdout, "lmv_foreign_value: 0x"); + for (i = 0; i < len2 - offsetof(typeof(*lfm), lfm_value); i++) + fprintf(stdout, "%02x", lfm->lfm_value[i]); + + fprintf(stdout, "\n"); + + return 0; +} diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index 27bbb01..23e5ffb 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -5661,6 +5661,106 @@ test_38() } run_test 38 "LFSCK does not break foreign file and reverse is also true" +test_39() +{ + [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.12.51) ]] && + skip "Need MDS version newer than 2.12.51" + + test_mkdir $DIR/$tdir + local uuid1=$(cat /proc/sys/kernel/random/uuid) + local uuid2=$(cat /proc/sys/kernel/random/uuid) + + # create foreign dir + $LFS mkdir --foreign=daos --xattr="${uuid1}@${uuid2}" --flags=0xda05 \ + $DIR/$tdir/${tdir}2 || + error "$DIR/$tdir/${tdir}2: create failed" + + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_magic:.*0x0CD50CD0" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA magic" + # lfm_length is LMV EA size - sizeof(lfm_magic) - sizeof(lfm_length) + # - sizeof(lfm_type) - sizeof(lfm_flags) + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_length:.*73" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA size" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_type:.*daos" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA type" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_flags:.*0x0000DA05" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA flags" + $LFS getdirstripe $DIR/$tdir/${tdir}2 | + grep "lfm_value.*${uuid1}@${uuid2}" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA value" + + # file create in dir should fail + touch $DIR/$tdir/${tdir}2/$tfile && + "$DIR/${tdir}2: file create should fail" + + # chmod should work + chmod 777 $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chmod failed" + + # chown should work + chown $RUNAS_ID:$RUNAS_GID $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chown failed" + + $START_NAMESPACE -r -A || error "Fail to start LFSCK for namespace" + + wait_all_targets_blocked namespace completed 1 + + # check that "global" namespace_repaired == 0 !!! + local repaired=$(do_facet mds1 \ + "$LCTL lfsck_query -t all -M ${FSNAME}-MDT0000 | + awk '/^namespace_repaired/ { print \\\$2 }'") + [ $repaired -eq 0 ] || + error "(2) Expect nothing to be repaired, but got: $repaired" + + $START_LAYOUT -A -r || error "Fail to start LFSCK for layout" + + wait_all_targets_blocked layout completed 2 + + # check that "global" layout_repaired == 0 !!! + local repaired=$(do_facet mds1 \ + "$LCTL lfsck_query -t all -M ${FSNAME}-MDT0000 | + awk '/^layout_repaired/ { print \\\$2 }'") + [ $repaired -eq 0 ] || + error "(2) Expect no layout repair, but got: $repaired" + + echo "post-lfsck checks of foreign dir" + + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_magic:.*0x0CD50CD0" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA magic" + # lfm_length is LMV EA size - sizeof(lfm_magic) - sizeof(lfm_length) + # - sizeof(lfm_type) - sizeof(lfm_flags) + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_length:.*73" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA size" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_type:.*daos" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA type" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_flags:.*0x0000DA05" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA flags" + $LFS getdirstripe $DIR/$tdir/${tdir}2 | + grep "lfm_value.*${uuid1}@${uuid2}" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA value" + + # file create in dir should fail + touch $DIR/$tdir/${tdir}2/$tfile && + "$DIR/${tdir}2: file create should fail" + + # chmod should work + chmod 777 $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chmod failed" + + # chown should work + chown $RUNAS_ID:$RUNAS_GID $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chown failed" + + #remove foreign dir + rmdir $DIR/$tdir/${tdir}2 || + error "$DIR/$tdir/${tdir}2: remove of foreign dir has failed" +} +run_test 39 "LFSCK does not break foreign dir and reverse is also true" + # restore MDS/OST size MDSSIZE=${SAVED_MDSSIZE} OSTSIZE=${SAVED_OSTSIZE} diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 4befadb..b39948b 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -2455,7 +2455,7 @@ test_27J() { $LFS getstripe -v $DIR/$tdir/${tfile}2 | grep "lfm_flags:.*0x0000DA08" || error "$DIR/$tdir/${tfile}2: invalid LOV EA foreign flags" - $LFS getstripe -v $DIR/$tdir/${tfile}2 | + $LFS getstripe $DIR/$tdir/${tfile}2 | grep "lfm_value:.*${uuid1}@${uuid2}" || error "$DIR/$tdir/${tfile}2: invalid LOV EA foreign value" @@ -2500,6 +2500,87 @@ test_27J() { } run_test 27J "basic ops on file with foreign LOV" +test_27K() { + [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.12.49) ]] && + skip "Need MDS version newer than 2.12.49" + + test_mkdir $DIR/$tdir + local uuid1=$(cat /proc/sys/kernel/random/uuid) + local uuid2=$(cat /proc/sys/kernel/random/uuid) + + # create foreign dir (raw way) + create_foreign_dir -d $DIR/$tdir/$tdir -x "${uuid1}@${uuid2}" -t 1 || + error "create_foreign_dir FAILED" + + # verify foreign dir (raw way) + parse_foreign_dir -d $DIR/$tdir/$tdir | + grep "lmv_foreign_magic:.*0xcd50cd0" || + error "$DIR/$tdir/$tfile: invalid LMV EA magic" + parse_foreign_dir -d $DIR/$tdir/$tdir | grep "lmv_xattr_size:.*89$" || + error "$DIR/$tdir/$tdir: invalid LMV EA size" + parse_foreign_dir -d $DIR/$tdir/$tdir | grep "lmv_foreign_type: 1$" || + error "$DIR/$tdir/$tdir: invalid LMV EA type" + parse_foreign_dir -d $DIR/$tdir/$tdir | grep "lmv_foreign_flags: 0$" || + error "$DIR/$tdir/$tdir: invalid LMV EA flags" + local lmv=$(parse_foreign_dir -d $DIR/$tdir/$tdir | + grep "lmv_foreign_value: 0x" | + sed 's/lmv_foreign_value: 0x//') + local lmv2=$(echo -n "${uuid1}@${uuid2}" | od -A n -t x1 -w160 | + sed 's/ //g') + [[ $lmv == $lmv2 ]] || error "$DIR/$tdir/$tdir: invalid LMV EA value" + + # create foreign dir (lfs + API) + $LFS mkdir --foreign=daos --xattr="${uuid1}@${uuid2}" --flags=0xda05 \ + $DIR/$tdir/${tdir}2 || + error "$DIR/$tdir/${tdir}2: create failed" + + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_magic:.*0x0CD50CD0" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA magic" + # lfm_length is LMV EA size - sizeof(lfm_magic) - sizeof(lfm_length) + # - sizeof(lfm_type) - sizeof(lfm_flags) + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_length:.*73" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA size" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | grep "lfm_type:.*daos" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA type" + $LFS getdirstripe -v $DIR/$tdir/${tdir}2 | + grep "lfm_flags:.*0x0000DA05" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA flags" + $LFS getdirstripe $DIR/$tdir/${tdir}2 | + grep "lfm_value.*${uuid1}@${uuid2}" || + error "$DIR/$tdir/${tdir}2: invalid LMV EA value" + + # file create in dir should fail + touch $DIR/$tdir/$tdir/$tfile && "$DIR/$tdir: file create should fail" + touch $DIR/$tdir/${tdir}2/$tfile && + "$DIR/${tdir}2: file create should fail" + + # chmod should work + chmod 777 $DIR/$tdir/$tdir || + error "$DIR/$tdir: chmod failed" + chmod 777 $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chmod failed" + + # chown should work + chown $RUNAS_ID:$RUNAS_GID $DIR/$tdir/$tdir || + error "$DIR/$tdir: chown failed" + chown $RUNAS_ID:$RUNAS_GID $DIR/$tdir/${tdir}2 || + error "$DIR/${tdir}2: chown failed" + + # rename should work + mv $DIR/$tdir/$tdir $DIR/$tdir/${tdir}.new || + error "$DIR/$tdir/$tdir: rename of foreign dir has failed" + mv $DIR/$tdir/${tdir}2 $DIR/$tdir/${tdir}2.new || + error "$DIR/$tdir/${tdir}2: rename of foreign dir has failed" + + #remove foreign dir + rmdir $DIR/$tdir/${tdir}.new || + error "$DIR/$tdir/${tdir}.new: remove of foreign dir has failed" + rmdir $DIR/$tdir/${tdir}2.new || + error "$DIR/$tdir/${tdir}2.new: remove of foreign dir has failed" +} +run_test 27K "basic ops on dir with foreign LMV" + # createtest also checks that device nodes are created and # then visible correctly (#2091) test_28() { # bug 2091 diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index b897626..4550b62 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -256,7 +256,12 @@ static inline int lfs_mirror_split(int argc, char **argv) " fnv_1a_64 FNV-1a hash algorithm (default)\n" \ " all_char sum of characters % MDT_COUNT (not recommended)\n" \ "\tdefault_stripe: set default dirstripe of the directory\n" \ - "\tmode: the file access permission of the directory (octal)\n" + "\tmode: the file access permission of the directory (octal)\n" \ + "To create dir with a foreign (free format) layout :\n" \ + "setdirstripe|mkdir --foreign[=] -x|-xattr " \ + "[--mode|-m mode] [--flags ] \n" \ + "\tmode: the mode of the directory\n" \ + "\tforeign_type: none or daos\n" /** * command_t mirror_cmdlist - lfs mirror commands. @@ -373,7 +378,7 @@ command_t cmdlist[] = { "usage: getdirstripe [--mdt-count|-c] [--mdt-index|-m|-i]\n" " [--mdt-hash|-H] [--obd|-O ]\n" " [--recursive|-r] [--yaml|-y]\n" - " [--default|-D] ..."}, + " [--verbose|-v] [--default|-D] ..."}, {"mkdir", lfs_setdirstripe, 0, "To create a striped directory on a specified MDT. This can only\n" "be done on MDT0 with the right of administrator.\n" @@ -402,6 +407,7 @@ command_t cmdlist[] = { " [[!] --gid|-g|--group|-G |]\n" " [[!] --uid|-u|--user|-U |] [[!] --pool ]\n" " [[!] --projid ]\n" + " [[!] --foreign[=]]\n" " [[!] --layout|-L released,raid0,mdt]\n" " [[!] --foreign[=]]\n" " [[!] --component-count [+-]]\n" @@ -637,19 +643,19 @@ static int check_hashtype(const char *hashtype) return 0; } -static uint32_t lov_check_foreign_type_name(const char *foreign_type_name) +static uint32_t check_foreign_type_name(const char *foreign_type_name) { uint32_t i; - for (i = 0; i < LOV_FOREIGN_TYPE_UNKNOWN; i++) { - if (lov_foreign_type[i].lft_name == NULL) + for (i = 0; i < LU_FOREIGN_TYPE_UNKNOWN; i++) { + if (lu_foreign_types[i].lft_name == NULL) break; if (strcmp(foreign_type_name, - lov_foreign_type[i].lft_name) == 0) - return lov_foreign_type[i].lft_type; + lu_foreign_types[i].lft_name) == 0) + return lu_foreign_types[i].lft_type; } - return LOV_FOREIGN_TYPE_UNKNOWN; + return LU_FOREIGN_TYPE_UNKNOWN; } static const char *error_loc = "syserror"; @@ -2630,7 +2636,7 @@ static int lfs_setstripe_internal(int argc, char **argv, char *template = NULL; bool foreign_mode = false; char *xattr = NULL; - uint32_t type = LOV_FOREIGN_TYPE_NONE, flags = 0; + uint32_t type = LU_FOREIGN_TYPE_NONE, flags = 0; char *mode_opt = NULL; mode_t previous_umask = 0; mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; @@ -2822,8 +2828,8 @@ static int lfs_setstripe_internal(int argc, char **argv, type = strtoul(optarg, &end, 0); if (*end) { /* check name */ - type = lov_check_foreign_type_name(optarg); - if (type == LOV_FOREIGN_TYPE_UNKNOWN) { + type = check_foreign_type_name(optarg); + if (type == LU_FOREIGN_TYPE_UNKNOWN) { fprintf(stderr, "%s %s: unrecognized foreign type '%s'\n", progname, argv[0], @@ -2840,7 +2846,7 @@ static int lfs_setstripe_internal(int argc, char **argv, mode = strtoul(mode_opt, &end, 8); if (*end != '\0') { fprintf(stderr, - "%s %s: bad MODE '%s'\n", + "%s %s: bad mode '%s'\n", progname, argv[0], mode_opt); return CMD_HELP; } @@ -3497,7 +3503,7 @@ static int lfs_setstripe_internal(int argc, char **argv, } } - if (mode_opt != NULL && previous_umask != 0) + if (mode_opt != NULL) umask(previous_umask); free(param); @@ -3697,6 +3703,8 @@ static int lfs_find(int argc, char **argv) { .val = 'E', .name = "component-end", .has_arg = required_argument }, /* find { .val = 'F', .name = "fid", .has_arg = no_argument }, */ + { .val = LFS_LAYOUT_FOREIGN_OPT, + .name = "foreign", .has_arg = optional_argument}, { .val = 'g', .name = "gid", .has_arg = required_argument }, { .val = 'G', .name = "group", .has_arg = required_argument }, { .val = 'H', .name = "mdt-hash", .has_arg = required_argument }, @@ -3896,30 +3904,6 @@ static int lfs_find(int argc, char **argv) param.fp_mirror_state = state; } break; - case LFS_LAYOUT_FOREIGN_OPT: { - /* all types by default */ - uint32_t type = LOV_FOREIGN_TYPE_UNKNOWN; - - if (optarg != NULL) { - /* check pure numeric */ - type = strtoul(optarg, &endptr, 0); - if (*endptr) { - /* check name */ - type = lov_check_foreign_type_name(optarg); - if (type == LOV_FOREIGN_TYPE_UNKNOWN) { - fprintf(stderr, - "%s %s: unrecognized foreign type '%s'\n", - progname, argv[0], - optarg); - return CMD_HELP; - } - } - } - param.fp_foreign_type = type; - param.fp_check_foreign = 1; - param.fp_exclude_foreign = !!neg_opt; - break; - } case 'c': if (optarg[0] == '+') { param.fp_stripe_count_sign = -1; @@ -3968,6 +3952,30 @@ static int lfs_find(int argc, char **argv) param.fp_check_comp_end = 1; param.fp_exclude_comp_end = !!neg_opt; break; + case LFS_LAYOUT_FOREIGN_OPT: { + /* all types by default */ + uint32_t type = LU_FOREIGN_TYPE_UNKNOWN; + + if (optarg != NULL) { + /* check pure numeric */ + type = strtoul(optarg, &endptr, 0); + if (*endptr) { + /* check name */ + type = check_foreign_type_name(optarg); + if (type == LU_FOREIGN_TYPE_UNKNOWN) { + fprintf(stderr, + "%s %s: unknown foreign type '%s'\n", + progname, argv[0], + optarg); + return CMD_HELP; + } + } + } + param.fp_foreign_type = type; + param.fp_check_foreign = 1; + param.fp_exclude_foreign = !!neg_opt; + break; + } case 'g': case 'G': rc = name2gid(¶m.fp_gid, optarg); @@ -4707,6 +4715,7 @@ static int lfs_getdirstripe(int argc, char **argv) { .val = 'O', .name = "obd", .has_arg = required_argument }, { .val = 'r', .name = "recursive", .has_arg = no_argument }, { .val = 'T', .name = "mdt-count", .has_arg = no_argument }, + { .val = 'v', .name = "verbose", .has_arg = no_argument }, { .val = 'y', .name = "yaml", .has_arg = no_argument }, { .name = NULL } }; int c, rc; @@ -4714,7 +4723,7 @@ static int lfs_getdirstripe(int argc, char **argv) param.fp_get_lmv = 1; while ((c = getopt_long(argc, argv, - "cDHimO:rtTy", long_opts, NULL)) != -1) + "cDHimO:rtTvy", long_opts, NULL)) != -1) { switch (c) { case 'c': @@ -4748,10 +4757,15 @@ static int lfs_getdirstripe(int argc, char **argv) case 'r': param.fp_recursive = 1; break; + case 'v': + param.fp_verbose |= VERBOSE_DETAIL; + break; case 'y': param.fp_yaml = 1; break; default: + fprintf(stderr, "%s %s: unrecognized option '%s'\n", + progname, argv[0], argv[optind - 1]); return CMD_HELP; } } @@ -5126,6 +5140,9 @@ static int lfs_setdirstripe(int argc, char **argv) struct ll_statfs_buf *lsb = NULL; char mntdir[PATH_MAX] = ""; bool auto_distributed = false; + bool foreign_mode = false; + char *xattr = NULL; + __u32 type = LU_FOREIGN_TYPE_DAOS, flags = 0; struct option long_opts[] = { { .val = 'c', .name = "count", .has_arg = required_argument }, @@ -5133,6 +5150,10 @@ static int lfs_setdirstripe(int argc, char **argv) { .val = 'd', .name = "delete", .has_arg = no_argument }, { .val = 'D', .name = "default", .has_arg = no_argument }, { .val = 'D', .name = "default_stripe", .has_arg = no_argument }, + { .val = LFS_LAYOUT_FLAGS_OPT, + .name = "flags", .has_arg = required_argument }, + { .val = LFS_LAYOUT_FOREIGN_OPT, + .name = "foreign", .has_arg = optional_argument}, { .val = 'H', .name = "mdt-hash", .has_arg = required_argument }, #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 17, 53, 0) { .val = 'i', .name = "mdt-index", .has_arg = required_argument }, @@ -5150,12 +5171,13 @@ static int lfs_setdirstripe(int argc, char **argv) #endif { .val = 'T', .name = "mdt-count", .has_arg = required_argument }, /* setstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */ + { .val = 'x', .name = "xattr", .has_arg = required_argument }, { .name = NULL } }; setstripe_args_init(&lsa); - while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:", long_opts, - NULL)) >= 0) { + while ((c = getopt_long(argc, argv, "c:dDi:H:m:o:t:T:x:", + long_opts, NULL)) >= 0) { switch (c) { case 0: /* Long options. */ @@ -5177,6 +5199,33 @@ static int lfs_setdirstripe(int argc, char **argv) case 'D': default_stripe = true; break; + case LFS_LAYOUT_FOREIGN_OPT: + if (optarg != NULL) { + /* check pure numeric */ + type = strtoul(optarg, &end, 0); + if (*end) { + /* check name */ + type = check_foreign_type_name(optarg); + if (type == LU_FOREIGN_TYPE_UNKNOWN) { + fprintf(stderr, + "%s %s: unknown foreign type '%s'\n", + progname, argv[0], + optarg); + return CMD_HELP; + } + } + } + foreign_mode = true; + break; + case LFS_LAYOUT_FLAGS_OPT: + flags = strtoul(optarg, &end, 16); + if (*end != '\0') { + fprintf(stderr, + "%s %s: bad flags '%s'\n", + progname, argv[0], optarg); + return CMD_HELP; + } + break; #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) case 't': fprintf(stderr, "warning: '--hash-type' and '-t' " @@ -5223,6 +5272,9 @@ static int lfs_setdirstripe(int argc, char **argv) case 'o': mode_opt = optarg; break; + case 'x': + xattr = optarg; + break; default: fprintf(stderr, "%s %s: unrecognized option '%s'\n", progname, argv[0], argv[optind - 1]); @@ -5236,8 +5288,30 @@ static int lfs_setdirstripe(int argc, char **argv) return CMD_HELP; } + if (xattr && !foreign_mode) { + /* only print a warning as this is armless and will be + * ignored + */ + fprintf(stderr, + "%s %s: xattr has been specified for non-foreign layout\n", + progname, argv[0]); + } else if (foreign_mode && !xattr) { + fprintf(stderr, + "%s %s: xattr must be provided in foreign mode\n", + progname, argv[0]); + return CMD_HELP; + } + + if (foreign_mode && (delete || default_stripe || lsa.lsa_nr_tgts || + lsa.lsa_tgts || setstripe_args_specified(&lsa))) { + fprintf(stderr, + "%s %s: only --xattr/--flags/--mode options are valid with --foreign\n", + progname, argv[0]); + return CMD_HELP; + } + if (!delete && lsa.lsa_stripe_off == LLAPI_LAYOUT_DEFAULT && - lsa.lsa_stripe_count == LLAPI_LAYOUT_DEFAULT) { + lsa.lsa_stripe_count == LLAPI_LAYOUT_DEFAULT && !foreign_mode) { fprintf(stderr, "%s %s: stripe offset and count must be specified\n", progname, argv[0]); @@ -5264,6 +5338,25 @@ static int lfs_setdirstripe(int argc, char **argv) previous_mode = umask(0); } + /* foreign LMV/dir case */ + if (foreign_mode) { + if (argc > optind + 1) { + fprintf(stderr, + "%s %s: cannot specify multiple foreign dirs\n", + progname, argv[0]); + return CMD_HELP; + } + + dname = argv[optind]; + result = llapi_dir_create_foreign(dname, mode, type, flags, + xattr); + if (result != 0) + fprintf(stderr, + "%s mkdir: can't create foreign dir '%s': %s\n", + progname, dname, strerror(-result)); + return result; + } + /* * initialize stripe parameters, in case param is converted to specific, * i.e, 'lfs mkdir -i -1 -c N', always allocate space for lsp_tgts. diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 7063285..d21f8da 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -86,11 +86,11 @@ char *mdt_hash_name[] = { "none", LMV_HASH_NAME_ALL_CHARS, LMV_HASH_NAME_FNV_1A_64 }; -struct lustre_foreign_type lov_foreign_type[] = { - {.lft_type = LOV_FOREIGN_TYPE_NONE, .lft_name = "none"}, - {.lft_type = LOV_FOREIGN_TYPE_DAOS, .lft_name = "daos"}, +struct lustre_foreign_type lu_foreign_types[] = { + {.lft_type = LU_FOREIGN_TYPE_NONE, .lft_name = "none"}, + {.lft_type = LU_FOREIGN_TYPE_DAOS, .lft_name = "daos"}, /* must be the last element */ - {.lft_type = LOV_FOREIGN_TYPE_UNKNOWN, .lft_name = NULL} + {.lft_type = LU_FOREIGN_TYPE_UNKNOWN, .lft_name = NULL} /* array max dimension must be <= UINT32_MAX */ }; @@ -1161,6 +1161,106 @@ out: return rc; } +/** + * Create a foreign directory. + * + * \param name the name of the directory to be created + * \param mode permission of the file if it is created, see mode in open(2) + * \param type foreign type to be set in LMV EA + * \param flags foreign flags to be set in LMV EA + * \param value foreign pattern to be set in LMV EA + * + * \retval 0 on success + * \retval negative errno on failure + */ +int llapi_dir_create_foreign(const char *name, mode_t mode, __u32 type, + __u32 flags, const char *value) +{ + struct lmv_foreign_md *lfm = NULL; + size_t lfm_size, len; + struct obd_ioctl_data data = { 0 }; + char rawbuf[8192]; + char *buf = rawbuf; + char *dirpath = NULL; + char *namepath = NULL; + char *dir; + char *filename; + int fd, rc; + + len = strlen(value); + if (len > XATTR_SIZE_MAX - offsetof(struct lmv_foreign_md, lfm_value) || + len <= 0) { + rc = -EINVAL; + llapi_error(LLAPI_MSG_ERROR, rc, "invalid LOV EA length %zu " + "(must be 0 < len < %zu)\n", len, + XATTR_SIZE_MAX - + offsetof(struct lmv_foreign_md, lfm_value)); + return rc; + } + lfm_size = len + offsetof(struct lmv_foreign_md, lfm_value); + lfm = calloc(1, lfm_size); + if (lfm == NULL) + return -ENOMEM; + + dirpath = strdup(name); + if (!dirpath) { + free(lfm); + return -ENOMEM; + } + + namepath = strdup(name); + if (!namepath) { + free(dirpath); + free(lfm); + return -ENOMEM; + } + + lfm->lfm_magic = LMV_MAGIC_FOREIGN; + lfm->lfm_length = len; + lfm->lfm_type = type; + lfm->lfm_flags = flags; + strncpy(lfm->lfm_value, value, len); + + filename = basename(namepath); + dir = dirname(dirpath); + + data.ioc_inlbuf1 = (char *)filename; + data.ioc_inllen1 = strlen(filename) + 1; + data.ioc_inlbuf2 = (char *)lfm; + data.ioc_inllen2 = lfm_size; + data.ioc_type = mode; + rc = llapi_ioctl_pack(&data, &buf, sizeof(rawbuf)); + if (rc) { + llapi_error(LLAPI_MSG_ERROR, rc, + "error: LL_IOC_LMV_SETSTRIPE pack failed '%s'.", + name); + goto out; + } + + fd = open(dir, O_DIRECTORY | O_RDONLY); + if (fd < 0) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "unable to open '%s'", name); + goto out; + } + + if (ioctl(fd, LL_IOC_LMV_SETSTRIPE, buf)) { + char *errmsg = "stripe already set"; + + rc = -errno; + if (errno != EEXIST && errno != EALREADY) + errmsg = strerror(errno); + + llapi_err_noerrno(LLAPI_MSG_ERROR, + "dirstripe error on '%s': %s", name, errmsg); + } + close(fd); +out: + free(namepath); + free(dirpath); + free(lfm); + return rc; +} int llapi_dir_create_pool(const char *name, int mode, int stripe_offset, int stripe_count, int stripe_pattern, @@ -1835,7 +1935,27 @@ again: int stripe_count; int lmv_size; - stripe_count = (__u32)param->fp_lmv_md->lum_stripe_count; + /* if foreign LMV case, fake stripes number */ + if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { + struct lmv_foreign_md *lfm; + + lfm = (struct lmv_foreign_md *)param->fp_lmv_md; + if (lfm->lfm_length < XATTR_SIZE_MAX - + offsetof(typeof(*lfm), lfm_value)) { + uint32_t size = lfm->lfm_length + + offsetof(typeof(*lfm), lfm_value); + + stripe_count = lmv_foreign_to_md_stripes(size); + } else { + llapi_error(LLAPI_MSG_ERROR, -EINVAL, + "error: invalid %d foreign size " + "returned from ioctl", + lfm->lfm_length); + return -EINVAL; + } + } else { + stripe_count = param->fp_lmv_md->lum_stripe_count; + } if (stripe_count <= param->fp_lmv_stripe_count) return ret; @@ -3597,18 +3717,18 @@ static void lov_dump_plain_user_lmm(struct find_param *param, char *path, } } -static uint32_t lov_check_foreign_type(uint32_t foreign_type) +static uint32_t check_foreign_type(uint32_t foreign_type) { uint32_t i; - for (i = 0; i < LOV_FOREIGN_TYPE_UNKNOWN; i++) { - if (lov_foreign_type[i].lft_name == NULL) + for (i = 0; i < LU_FOREIGN_TYPE_UNKNOWN; i++) { + if (lu_foreign_types[i].lft_name == NULL) break; - if (foreign_type == lov_foreign_type[i].lft_type) + if (foreign_type == lu_foreign_types[i].lft_type) return i; } - return LOV_FOREIGN_TYPE_UNKNOWN; + return LU_FOREIGN_TYPE_UNKNOWN; } static void lov_dump_foreign_lmm(struct find_param *param, char *path, @@ -3621,7 +3741,7 @@ static void lov_dump_foreign_lmm(struct find_param *param, char *path, llapi_printf(LLAPI_MSG_NORMAL, "%s\n", path); if (param->fp_verbose & VERBOSE_DETAIL) { - uint32_t type = lov_check_foreign_type(lfm->lfm_type); + uint32_t type = check_foreign_type(lfm->lfm_type); llapi_printf(LLAPI_MSG_NORMAL, "lfm_magic: 0x%08X\n", lfm->lfm_magic); @@ -3629,16 +3749,47 @@ static void lov_dump_foreign_lmm(struct find_param *param, char *path, lfm->lfm_length); llapi_printf(LLAPI_MSG_NORMAL, "lfm_type: 0x%08X", lfm->lfm_type); - if (type < LOV_FOREIGN_TYPE_UNKNOWN) + if (type < LU_FOREIGN_TYPE_UNKNOWN) llapi_printf(LLAPI_MSG_NORMAL, " (%s)\n", - lov_foreign_type[type].lft_name); + lu_foreign_types[type].lft_name); else llapi_printf(LLAPI_MSG_NORMAL, " (unknown)\n"); llapi_printf(LLAPI_MSG_NORMAL, "lfm_flags: 0x%08X\n", lfm->lfm_flags); } + llapi_printf(LLAPI_MSG_NORMAL, "lfm_value: '%.*s'\n", + lfm->lfm_length, lfm->lfm_value); + llapi_printf(LLAPI_MSG_NORMAL, "\n"); +} + +static void lmv_dump_foreign_lmm(struct find_param *param, char *path, + enum lov_dump_flags flags) +{ + struct lmv_foreign_md *lfm = (struct lmv_foreign_md *)param->fp_lmv_md; + bool yaml = flags & LDF_YAML; + + if (!yaml && param->fp_depth && path) + llapi_printf(LLAPI_MSG_NORMAL, "%s\n", path); + if (param->fp_verbose & VERBOSE_DETAIL) { + uint32_t type = check_foreign_type(lfm->lfm_type); + + llapi_printf(LLAPI_MSG_NORMAL, "lfm_magic: 0x%08X\n", + lfm->lfm_magic); + llapi_printf(LLAPI_MSG_NORMAL, "lfm_length: %u\n", + lfm->lfm_length); + llapi_printf(LLAPI_MSG_NORMAL, "lfm_type: 0x%08X", + lfm->lfm_type); + if (type < LU_FOREIGN_TYPE_UNKNOWN) + llapi_printf(LLAPI_MSG_NORMAL, " (%s)\n", + lu_foreign_types[type].lft_name); + else + llapi_printf(LLAPI_MSG_NORMAL, " (unknown)\n"); + + llapi_printf(LLAPI_MSG_NORMAL, "lfm_flags: 0x%08X\n", + lfm->lfm_flags); + } llapi_printf(LLAPI_MSG_NORMAL, "lfm_value: '%.*s'\n", lfm->lfm_length, lfm->lfm_value); llapi_printf(LLAPI_MSG_NORMAL, "\n"); @@ -3684,6 +3835,9 @@ static void llapi_lov_dump_user_lmm(struct find_param *param, char *path, case LOV_USER_MAGIC_COMP_V1: lov_dump_comp_v1(param, path, flags); break; + case LMV_MAGIC_FOREIGN: + lmv_dump_foreign_lmm(param, path, flags); + break; default: llapi_printf(LLAPI_MSG_NORMAL, "unknown lmm_magic: %#x " "(expecting one of %#x %#x %#x %#x)\n", @@ -4022,11 +4176,27 @@ static int find_check_foreign(struct find_param *param) lfm = (void *)¶m->fp_lmd->lmd_lmm; if (lfm->lfm_magic != LOV_USER_MAGIC_FOREIGN) { - if (param->fp_foreign_type == LOV_FOREIGN_TYPE_UNKNOWN) + if (param->fp_foreign_type == LU_FOREIGN_TYPE_UNKNOWN) + return param->fp_exclude_foreign ? 1 : -1; + return -1; + } else { + if (param->fp_foreign_type == LU_FOREIGN_TYPE_UNKNOWN || + lfm->lfm_type == param->fp_foreign_type) + return param->fp_exclude_foreign ? -1 : 1; + return param->fp_exclude_foreign ? 1 : -1; + } + } + + if (S_ISDIR(param->fp_lmd->lmd_st.st_mode)) { + struct lmv_foreign_md *lfm; + + lfm = (void *)param->fp_lmv_md; + if (lfm->lfm_magic != LMV_MAGIC_FOREIGN) { + if (param->fp_foreign_type == LU_FOREIGN_TYPE_UNKNOWN) return param->fp_exclude_foreign ? 1 : -1; return -1; } else { - if (param->fp_foreign_type == LOV_FOREIGN_TYPE_UNKNOWN || + if (param->fp_foreign_type == LU_FOREIGN_TYPE_UNKNOWN || lfm->lfm_type == param->fp_foreign_type) return param->fp_exclude_foreign ? -1 : 1; return param->fp_exclude_foreign ? 1 : -1; @@ -4274,11 +4444,22 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, decision = 0; if (decision == 0) { - if (param->fp_check_mdt_count || param->fp_check_hash_type) { + if (dir && (param->fp_check_mdt_count || + param->fp_check_hash_type || param->fp_check_foreign)) { param->fp_get_lmv = 1; ret = cb_get_dirstripe(path, dir, param); - if (ret != 0) + if (ret != 0) { + /* XXX this works to decide for foreign + * criterion only + */ + if (errno == ENODATA && + param->fp_check_foreign) { + if (param->fp_exclude_foreign) + goto foreign; + goto decided; + } return ret; + } } param->fp_lmd->lmd_lmm.lmm_magic = 0; @@ -4393,6 +4574,11 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, } if (param->fp_check_mdt_count) { + if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { + decision = -1; + goto decided; + } + decision = find_value_cmp( param->fp_lmv_md->lum_stripe_count, param->fp_mdt_count, @@ -4411,6 +4597,11 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, if (param->fp_check_hash_type) { __u32 found; + if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { + decision = -1; + goto decided; + } + found = param->fp_lmv_md->lum_hash_type & param->fp_hash_type; if ((found && param->fp_exclude_hash_type) || (!found && !param->fp_exclude_hash_type)) { @@ -4584,6 +4775,7 @@ obd_matches: goto decided; } +foreign: llapi_printf(LLAPI_MSG_NORMAL, "%s", path); if (param->fp_zero_end) llapi_printf(LLAPI_MSG_NORMAL, "%c", '\0'); diff --git a/lustre/utils/liblustreapi.map b/lustre/utils/liblustreapi.map index bdb832a..a0d16b2 100644 --- a/lustre/utils/liblustreapi.map +++ b/lustre/utils/liblustreapi.map @@ -6,7 +6,7 @@ liblustreapi_initialized; l_ioctl; mdt_hash_name; - lov_foreign_type; + lu_foreign_types; Parser_*; register_ioc_*; local: -- 1.8.3.1