From da3536da22aba6b61da2c82b71e4eec423425446 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 9 Dec 2024 12:53:09 +0100 Subject: [PATCH] LU-18357 ptlrpc: Use nodemap IAM for persisting filesets Currently, two separate commands exist to set a fileset and both behave differently. "lctl set_param (-P)" works as expected. On the other hand, "lctl nodemap_set_fileset" updates the fileset only on the local node and is not persistent. It could also be run from other server nodes and not only from the MGS. This behavior is inconsistent with other nodemap commands. Ideally, we would want to use the nodemap IAM records for persisting all nodemap properties without relying on secondary mechanisms, such as the params llog which is currently used for filesets. This patch adds support for persistence across restarts for "lctl nodemap_set_fileset" commands. Filesets are split into 28 byte path fragments which are stored alongside 4 byte fragment metadata into one nodemap IAM record (which is 32 bytes). For this, fileset fragments are stored in a "defined" subid range within the NODEMAP_CLUSTER_IDX type. In anticipation of supporting multiple filesets, the first fileset subid is reserved for a fileset header. The subid range is set to support at maximum 256 nodemap records for fileset fragments. 256 entries support enough fragments to hold a PATH_MAX fileset in total. Only needed fragments are inserted into the nodemap IAM. To support seamless updates from fileset llog to IAM versions, a nodemap flag was added which allows existing "lctl set_param -P nodemap..fileset" to be applied outside the nodemap IAM as long as no IAM records were set. Filesets from the llog are not migrated to the IAM but are still compatbiel after an update. "lctl set_param nodemap..fileset" is deprecated and uses the IAM once fileset IAM records are in use. Sanity-sec "test_27a" was modified to test fileset persistence. Signed-off-by: Marc Vef Change-Id: Ic2e5e8026261867b7f0e90ea4ee2287251c503f7 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/56757 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Sebastien Buisson Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- lustre/include/lustre_nodemap.h | 8 +- lustre/include/uapi/linux/lustre/lustre_disk.h | 25 + lustre/ptlrpc/nodemap_handler.c | 139 +++++- lustre/ptlrpc/nodemap_internal.h | 19 + lustre/ptlrpc/nodemap_lproc.c | 2 +- lustre/ptlrpc/nodemap_storage.c | 660 ++++++++++++++++++++++--- lustre/ptlrpc/wiretest.c | 19 + lustre/tests/sanity-sec.sh | 97 +++- lustre/utils/lustre_param.c | 43 ++ lustre/utils/wirecheck.c | 12 + lustre/utils/wiretest.c | 19 + 11 files changed, 934 insertions(+), 109 deletions(-) diff --git a/lustre/include/lustre_nodemap.h b/lustre/include/lustre_nodemap.h index 55564f6..4eaf632 100644 --- a/lustre/include/lustre_nodemap.h +++ b/lustre/include/lustre_nodemap.h @@ -58,7 +58,8 @@ struct lu_nodemap { nmf_enable_audit:1, nmf_forbid_encryption:1, nmf_readonly_mount:1, - nmf_deny_mount:1; + nmf_deny_mount:1, + nmf_fileset_use_iam:1; /* bitmap for mapping type */ enum nodemap_mapping_modes nmf_map_mode; /* bitmap for rbac, enum nodemap_rbac_roles */ @@ -96,7 +97,7 @@ struct lu_nodemap { struct hlist_node nm_hash; struct nodemap_pde *nm_pde_data; /* fileset the nodes of this nodemap are restricted to */ - char nm_fileset[PATH_MAX+1]; + char nm_fileset[PATH_MAX + 1]; /* information about the expected SELinux policy on the nodes */ char nm_sepol[LUSTRE_NODEMAP_SEPOL_LENGTH + 1]; @@ -161,7 +162,8 @@ int nodemap_add_idmap(const char *nodemap_name, enum nodemap_id_type id_type, const __u32 map[2]); int nodemap_del_idmap(const char *nodemap_name, enum nodemap_id_type id_type, const __u32 map[2]); -int nodemap_set_fileset(const char *name, const char *fileset, bool checkperm); +int nodemap_set_fileset(const char *name, const char *fileset, bool checkperm, + bool ioctl_op); char *nodemap_get_fileset(const struct lu_nodemap *nodemap); int nodemap_set_sepol(const char *name, const char *sepol, bool checkperm); const char *nodemap_get_sepol(const struct lu_nodemap *nodemap); diff --git a/lustre/include/uapi/linux/lustre/lustre_disk.h b/lustre/include/uapi/linux/lustre/lustre_disk.h index da56d26..516e414 100644 --- a/lustre/include/uapi/linux/lustre/lustre_disk.h +++ b/lustre/include/uapi/linux/lustre/lustre_disk.h @@ -259,6 +259,7 @@ enum nm_flag_bits { enum nm_flag2_bits { NM_FL2_READONLY_MOUNT = 0x1, NM_FL2_DENY_MOUNT = 0x2, + NM_FL2_FILESET_USE_IAM = 0x4, }; /* Nodemap records, uses 32 byte record length. @@ -332,6 +333,19 @@ struct nodemap_offset_rec { __u32 nor_padding2; }; +/* fileset fragment length for each nodemap record: 28 bytes for fragments */ +#define LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE \ + (sizeof(struct nodemap_cluster_rec) - (2 * sizeof(__u16))) +/* fileset subid range to support a PATH_MAX characters fileset and header */ +#define LUSTRE_NODEMAP_FILESET_SUBID_RANGE 256 + +struct nodemap_fileset_rec { + /* 28 bytes for fileset path fragment */ + char nfr_path_fragment[LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE]; + __u16 nfr_fragment_id; /* fileset fragment id */ + __u16 nfr_padding1; /* zeroed, reserved for multi fset id */ +}; + union nodemap_rec { struct nodemap_cluster_rec ncr; struct nodemap_range_rec nrr; @@ -340,6 +354,7 @@ union nodemap_rec { struct nodemap_global_rec ngr; struct nodemap_cluster_roles_rec ncrr; struct nodemap_offset_rec nor; + struct nodemap_fileset_rec nfr; }; /* sub-keys for records of type NODEMAP_CLUSTER_IDX */ @@ -347,6 +362,16 @@ enum nodemap_cluster_rec_subid { NODEMAP_CLUSTER_REC = 0, /* nodemap_cluster_rec */ NODEMAP_CLUSTER_ROLES = 1, /* nodemap_cluster_roles_rec */ NODEMAP_CLUSTER_OFFSET = 2, /* UID/GID/PROJID offset for a nm cluster */ + /* + * A fileset may not fit in a single nodemap_fileset_rec and uses at max + * 256 fragments. The first subid (512) is currently unused and reserved + * for a future fileset header. + */ + NODEMAP_FILESET = 512, + /* + * Depending on its length, its fragments may use several subids + * in the range 512-767. + */ }; /* first 4 bits of the nodemap_id is the index type */ diff --git a/lustre/ptlrpc/nodemap_handler.c b/lustre/ptlrpc/nodemap_handler.c index f46d410..c0908ac 100644 --- a/lustre/ptlrpc/nodemap_handler.c +++ b/lustre/ptlrpc/nodemap_handler.c @@ -966,6 +966,7 @@ static void nodemap_inherit_properties(struct lu_nodemap *dst, dst->nmf_readonly_mount = 0; dst->nmf_rbac = NODEMAP_RBAC_ALL; dst->nmf_deny_mount = 0; + dst->nmf_fileset_use_iam = 0; dst->nm_squash_uid = NODEMAP_NOBODY_UID; dst->nm_squash_gid = NODEMAP_NOBODY_GID; @@ -988,6 +989,8 @@ static void nodemap_inherit_properties(struct lu_nodemap *dst, dst->nmf_readonly_mount = src->nmf_readonly_mount; dst->nmf_rbac = src->nmf_rbac; dst->nmf_deny_mount = src->nmf_deny_mount; + dst->nmf_fileset_use_iam = 0; + dst->nm_squash_uid = src->nm_squash_uid; dst->nm_squash_gid = src->nm_squash_gid; dst->nm_squash_projid = src->nm_squash_projid; @@ -1151,16 +1154,73 @@ out: EXPORT_SYMBOL(nodemap_del_range); /** - * set fileset on nodemap - * \param name nodemap to set fileset on + * Set a fileset on a nodemap in memory and the nodemap IAM records. + * If the nodemap is dynamic, the nodemap IAM update is transparently skipped in + * the nodemap_idx_fileset_* functions to update only the in-memory nodemap. + * + * \param nodemap the nodemap to set fileset on * \param fileset string containing fileset * \retval 0 on success + * \retval -EINVAL invalid fileset: Does not start with '/' + * \retval -ENAMETOOLONG fileset is too long + * \retval -EIO undo operation failed during IAM update + */ +static int nodemap_set_fileset_iam(struct lu_nodemap *nodemap, + const char *fileset) +{ + int rc = 0; + + if (strlen(fileset) > PATH_MAX) + RETURN(-ENAMETOOLONG); + + if (fileset[0] == '\0' || strcmp(fileset, "clear") == 0) { + rc = nodemap_idx_fileset_clear(nodemap); + if (rc == 0) + nodemap->nm_fileset[0] = '\0'; + } else if (fileset[0] != '/') { + rc = -EINVAL; + } else { + /* + * Only update the index if the fileset is already set. + * If it was set by the params llog, it is not set in the IAM. + */ + if (nodemap->nm_fileset[0] != '\0' && + nodemap->nmf_fileset_use_iam) { + rc = nodemap_idx_fileset_update( + nodemap, nodemap->nm_fileset, fileset); + } else { + rc = nodemap_idx_fileset_add(nodemap, fileset); + } + if (rc < 0) + GOTO(out, rc); + + memcpy(nodemap->nm_fileset, fileset, strlen(fileset) + 1); + } + + if (!nodemap->nm_dyn && !nodemap->nmf_fileset_use_iam) { + nodemap->nmf_fileset_use_iam = 1; + rc = nodemap_idx_nodemap_update(nodemap); + } + +out: + return rc; +} + +/** + * Set a fileset on a nodemap. This is a local operation and not persistent. * - * set a fileset on the named nodemap + * This function is a remnant from when fileset updates were made through + * the params llog, which caused "lctl set_param" to be called on + * each server locally. For backward compatibility this functionality is kept. + * + * \param nodemap the nodemap to set fileset on + * \param fileset string containing fileset + * \retval 0 on success + * \retval -EINVAL invalid fileset: Does not start with '/' + * \retval -ENAMETOOLONG fileset is too long */ -static int nodemap_set_fileset_helper(struct nodemap_config *config, - struct lu_nodemap *nodemap, - const char *fileset) +static int nodemap_set_fileset_local(struct lu_nodemap *nodemap, + const char *fileset) { int rc = 0; @@ -1173,40 +1233,73 @@ static int nodemap_set_fileset_helper(struct nodemap_config *config, * won't clear fileset. * 'fileset=""' is still kept for compatibility reason. */ - if (fileset == NULL) - rc = -EINVAL; - else if (fileset[0] == '\0' || strcmp(fileset, "clear") == 0) + if (fileset[0] == '\0' || strcmp(fileset, "clear") == 0) { nodemap->nm_fileset[0] = '\0'; - else if (fileset[0] != '/') + } else if (fileset[0] != '/') { rc = -EINVAL; - else if (strscpy(nodemap->nm_fileset, fileset, - sizeof(nodemap->nm_fileset)) < 0) + + } else if (strscpy(nodemap->nm_fileset, fileset, + sizeof(nodemap->nm_fileset)) < 0) { rc = -ENAMETOOLONG; + /* function may be called from llog thread, so we CERROR here */ + CERROR("%s: fileset '%s' is too long: rc = %d\n", + nodemap->nm_name, fileset, rc); + } return rc; } -int nodemap_set_fileset(const char *name, const char *fileset, bool checkperm) +/** + * Set fileset on a named nodemap + * + * \param name name of the nodemap to set fileset on + * \param fileset string containing fileset + * \param checkperm true if permission check is required + * \param ioctl_op true if called from ioctl nodemap functions + * \retval 0 on success + */ +int nodemap_set_fileset(const char *name, const char *fileset, bool checkperm, + bool ioctl_op) { - struct lu_nodemap *nodemap = NULL; - int rc = 0; + struct lu_nodemap *nodemap = NULL; + int rc = 0; + + ENTRY; + + if (name == NULL || name[0] == '\0' || fileset == NULL) + RETURN(-EINVAL); mutex_lock(&active_config_lock); nodemap = nodemap_lookup(name); if (IS_ERR(nodemap)) { mutex_unlock(&active_config_lock); - GOTO(out, rc = PTR_ERR(nodemap)); + RETURN(PTR_ERR(nodemap)); } if (checkperm && !allow_op_on_nm(nodemap)) GOTO(out_unlock, rc = -EPERM); - rc = nodemap_set_fileset_helper(active_config, nodemap, fileset); + /* + * Previously filesets were made persistent through the params llog, + * which caused local fileset updates on the server nodes. Now, filesets + * are made persistent through the nodemap IAM records. Since we need to + * be backward-compatible, this function serves as a mechanism to + * support filesets saved in the llog as long as no IAM records were + * set. "nodemap->nmf_fileset_use_iam" controls the transition between + * both backends. Local updates are disabled once IAM records are used. + */ + if (ioctl_op) + rc = nodemap_set_fileset_iam(nodemap, fileset); + else if (!ioctl_op && !nodemap->nmf_fileset_use_iam) + rc = nodemap_set_fileset_local(nodemap, fileset); + else + rc = -EINVAL; out_unlock: mutex_unlock(&active_config_lock); nodemap_putref(nodemap); -out: + + EXIT; return rc; } EXPORT_SYMBOL(nodemap_set_fileset); @@ -1944,6 +2037,14 @@ int nodemap_del(const char *nodemap_name) } up_write(&active_config->nmc_range_tree_lock); + if (nodemap->nm_fileset[0] != '\0') { + rc2 = nodemap_idx_fileset_clear(nodemap); + if (rc2 < 0) + rc = rc2; + + nodemap->nm_fileset[0] = '\0'; + } + if (!nodemap->nm_dyn) { rc2 = nodemap_idx_nodemap_del(nodemap); if (rc2 < 0) @@ -2687,7 +2788,7 @@ static int cfg_nodemap_cmd(enum lcfg_command_type cmd, const char *nodemap_name, rc = -EINVAL; break; case LCFG_NODEMAP_SET_FILESET: - rc = nodemap_set_fileset(nodemap_name, param, true); + rc = nodemap_set_fileset(nodemap_name, param, true, true); break; case LCFG_NODEMAP_SET_SEPOL: rc = nodemap_set_sepol(nodemap_name, param, true); diff --git a/lustre/ptlrpc/nodemap_internal.h b/lustre/ptlrpc/nodemap_internal.h index fa177df..766c7df 100644 --- a/lustre/ptlrpc/nodemap_internal.h +++ b/lustre/ptlrpc/nodemap_internal.h @@ -65,6 +65,17 @@ struct lu_idmap { struct rb_node id_fs_to_client; }; +struct lu_nodemap_fileset_info { + /* nodemap id */ + __u32 nfi_nm_id; + /* starting subid of the fileset in the IAM */ + __u32 nfi_subid; + /* number of fileset fragments */ + __u32 nfi_fragment_cnt; + /* the fileset */ + const char *nfi_fileset; +}; + static inline enum nodemap_idx_type nm_idx_get_type(unsigned int id) { return id >> NM_TYPE_SHIFT; @@ -149,6 +160,14 @@ int nodemap_idx_cluster_roles_update(const struct lu_nodemap *nodemap); int nodemap_idx_cluster_roles_del(const struct lu_nodemap *nodemap); int nodemap_idx_offset_add(const struct lu_nodemap *nodemap); int nodemap_idx_offset_del(const struct lu_nodemap *nodemap); +int nodemap_idx_fileset_add(const struct lu_nodemap *nodemap, + const char *fileset); +int nodemap_idx_fileset_update(const struct lu_nodemap *nodemap, + const char *old_fileset, + const char *new_fileset); +int nodemap_idx_fileset_del(const struct lu_nodemap *nodemap, + const char *fileset); +int nodemap_idx_fileset_clear(const struct lu_nodemap *nodemap); int nodemap_idx_idmap_add(const struct lu_nodemap *nodemap, enum nodemap_id_type id_type, const __u32 map[2]); diff --git a/lustre/ptlrpc/nodemap_lproc.c b/lustre/ptlrpc/nodemap_lproc.c index f3165e2..92ddb0d 100644 --- a/lustre/ptlrpc/nodemap_lproc.c +++ b/lustre/ptlrpc/nodemap_lproc.c @@ -250,7 +250,7 @@ nodemap_fileset_seq_write(struct file *file, if (copy_from_user(nm_fileset, buffer, count)) GOTO(out, rc = -EFAULT); - rc = nodemap_set_fileset(m->private, nm_fileset, false); + rc = nodemap_set_fileset(m->private, nm_fileset, false, false); if (rc != 0) GOTO(out, rc = -EINVAL); diff --git a/lustre/ptlrpc/nodemap_storage.c b/lustre/ptlrpc/nodemap_storage.c index 44ecc82..08e3e0c 100644 --- a/lustre/ptlrpc/nodemap_storage.c +++ b/lustre/ptlrpc/nodemap_storage.c @@ -88,7 +88,8 @@ static void nodemap_cluster_rec_init(union nodemap_rec *nr, NM_FL_FORBID_ENCRYPT : 0); nr->ncr.ncr_flags2 = (nodemap->nmf_readonly_mount ? NM_FL2_READONLY_MOUNT : 0) | - (nodemap->nmf_deny_mount ? NM_FL2_DENY_MOUNT : 0); + (nodemap->nmf_deny_mount ? NM_FL2_DENY_MOUNT : 0) | + (nodemap->nmf_fileset_use_iam ? NM_FL2_FILESET_USE_IAM : 0); nr->ncr.ncr_padding1 = 0; nr->ncr.ncr_squash_projid = cpu_to_le32(nodemap->nm_squash_projid); nr->ncr.ncr_squash_uid = cpu_to_le32(nodemap->nm_squash_uid); @@ -120,6 +121,30 @@ static void nodemap_offset_rec_init(union nodemap_rec *nr, nor->nor_limit_projid = cpu_to_le32(nodemap->nm_offset_limit_projid); } +static int nodemap_cluster_fileset_rec_init(union nodemap_rec *nr, + const char *fileset, + unsigned int fragment_id, + unsigned int fragment_size) +{ + struct nodemap_fileset_rec *nfr = &nr->nfr; + unsigned int fset_offset; + int rc = 0; + + if (fragment_size > LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE || + fragment_size > strlen(fileset) + 1) { + rc = -ENAMETOOLONG; + CERROR("%s: Invalid fileset fragment size: rc = %d\n", fileset, + rc); + RETURN(rc); + } + + nfr->nfr_fragment_id = cpu_to_le16(fragment_id); + fset_offset = fragment_id * LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE; + memcpy(nfr->nfr_path_fragment, fileset + fset_offset, fragment_size); + + return rc; +} + static void nodemap_idmap_key_init(struct nodemap_key *nk, unsigned int nm_id, enum nodemap_id_type id_type, u32 id_client) @@ -306,14 +331,29 @@ out: return nm_obj; } -static int nodemap_idx_insert(const struct lu_env *env, - struct dt_object *idx, - const struct nodemap_key *nk, - const union nodemap_rec *nr) +/** + * Batch inserts a number of keys and records into the nodemap IAM. + * + * \param env execution environment + * \param idx index object to insert into + * \param nks array of keys to insert + * \param nrs array of records to insert + * \param count number of keys and records to insert + * \param inserted_out pointer to the number of records inserted. + * May be set to NULL if not needed. + * + * \retval negative errno if the insertion fails + */ +static int nodemap_idx_insert_batch(const struct lu_env *env, + struct dt_object *idx, + const struct nodemap_key *nks, + const union nodemap_rec *nrs, int count, + int *inserted_out) { struct thandle *th; struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev); - int rc; + int inserted = 0; + int rc, i; BUILD_BUG_ON(sizeof(union nodemap_rec) != 32); @@ -321,11 +361,12 @@ static int nodemap_idx_insert(const struct lu_env *env, if (IS_ERR(th)) RETURN(PTR_ERR(th)); - rc = dt_declare_insert(env, idx, - (const struct dt_rec *)nr, - (const struct dt_key *)nk, th); - if (rc != 0) - GOTO(out, rc); + for (i = 0; i < count; i++) { + rc = dt_declare_insert(env, idx, (const struct dt_rec *)&nrs[i], + (const struct dt_key *)&nks[i], th); + if (rc != 0) + GOTO(out, rc); + } rc = dt_declare_version_set(env, idx, th); if (rc != 0) @@ -337,25 +378,41 @@ static int nodemap_idx_insert(const struct lu_env *env, dt_write_lock(env, idx, 0); - rc = dt_insert(env, idx, (const struct dt_rec *)nr, - (const struct dt_key *)nk, th); + for (i = 0; i < count; i++) { + rc = dt_insert(env, idx, (const struct dt_rec *)&nrs[i], + (const struct dt_key *)&nks[i], th); + if (rc != 0) + break; + inserted++; + } nodemap_inc_version(env, idx, th); dt_write_unlock(env, idx); out: dt_trans_stop(env, dev, th); + if (inserted_out != NULL) + *inserted_out = inserted; + return rc; } +static int nodemap_idx_insert(const struct lu_env *env, + struct dt_object *idx, + const struct nodemap_key *nk, + const union nodemap_rec *nr) +{ + return nodemap_idx_insert_batch(env, idx, nk, nr, 1, NULL); +} + static int nodemap_idx_update(const struct lu_env *env, struct dt_object *idx, const struct nodemap_key *nk, const union nodemap_rec *nr) { - struct thandle *th; - struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev); - int rc = 0; + struct thandle *th; + struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev); + int rc = 0; th = dt_trans_create(env, dev); if (IS_ERR(th)) @@ -398,22 +455,40 @@ out: return rc; } -static int nodemap_idx_delete(const struct lu_env *env, - struct dt_object *idx, - const struct nodemap_key *nk, - const union nodemap_rec *unused) +/** + * Batch deletes a number of keys and records from the nodemap IAM. + * + * \param env execution environment + * \param idx index object to delete + * \param nks array of keys to delete + * \param nrs array of records to delete + * \param count number of keys to delete + * \param deleted_out pointer to the number of records deleted. + * May be set to NULL if not needed. + * + * \retval negative errno if the insertion fails. + * ENOENT if the key does not exist is ignored. + */ +static int nodemap_idx_delete_batch(const struct lu_env *env, + struct dt_object *idx, + const struct nodemap_key *nks, int count, + int *deleted_out) { - struct thandle *th; - struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev); - int rc = 0; + struct thandle *th; + struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev); + int deleted = 0; + int rc, i; th = dt_trans_create(env, dev); if (IS_ERR(th)) RETURN(PTR_ERR(th)); - rc = dt_declare_delete(env, idx, (const struct dt_key *)nk, th); - if (rc != 0) - GOTO(out, rc); + for (i = 0; i < count; i++) { + rc = dt_declare_delete(env, idx, (const struct dt_key *)&nks[i], + th); + if (rc != 0) + GOTO(out, rc); + } rc = dt_declare_version_set(env, idx, th); if (rc != 0) @@ -425,17 +500,35 @@ static int nodemap_idx_delete(const struct lu_env *env, dt_write_lock(env, idx, 0); - rc = dt_delete(env, idx, (const struct dt_key *)nk, th); + for (i = 0; i < count; i++) { + rc = dt_delete(env, idx, (const struct dt_key *)&nks[i], th); + if (rc == -ENOENT) + continue; + if (rc != 0) + break; - nodemap_inc_version(env, idx, th); + deleted++; + } + nodemap_inc_version(env, idx, th); dt_write_unlock(env, idx); out: dt_trans_stop(env, dev, th); + if (deleted_out != NULL) + *deleted_out = deleted; + return rc; } +static int nodemap_idx_delete(const struct lu_env *env, + struct dt_object *idx, + const struct nodemap_key *nk, + const union nodemap_rec *unused) +{ + return nodemap_idx_delete_batch(env, idx, nk, 1, NULL); +} + enum nm_add_update { NM_ADD = 0, NM_UPDATE = 1, @@ -656,6 +749,418 @@ int nodemap_idx_offset_del(const struct lu_nodemap *nodemap) RETURN(rc); } +/** + * Inserts fileset fragments (identified by struct lu_nodemap_fileset_info) into + * the nodemap IAM. + * + * \param fset_info fileset info to be inserted into the nodemap IAM + * \param env execution environment + * \param idx index object to insert into + * \param inserted_out pointer to the number of records inserted. + * May be set to NULL if not needed. + * + * \retval 0 on success + * \retval -EINVAL invalid input parameters + * \retval -ENOMEM memory allocation failure + */ +static int nodemap_idx_fileset_fragments_add( + const struct lu_nodemap_fileset_info *fset_info, + const struct lu_env *env, struct dt_object *idx, int *inserted_out) +{ + struct nodemap_key *nk_array; + union nodemap_rec *nr_array; + unsigned int size_remaining, fragment_size; + int i; + int rc = 0; + + if (fset_info == NULL || fset_info->nfi_fileset == NULL || + fset_info->nfi_fragment_cnt == 0 || + fset_info->nfi_fragment_cnt > + LUSTRE_NODEMAP_FILESET_SUBID_RANGE - 1) + RETURN(-EINVAL); + + OBD_ALLOC_PTR_ARRAY(nk_array, fset_info->nfi_fragment_cnt); + if (nk_array == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC_PTR_ARRAY(nr_array, fset_info->nfi_fragment_cnt); + if (nr_array == NULL) + GOTO(out_cleanup, rc = -ENOMEM); + + /* setup fileset fragment keys and records to be inserted */ + size_remaining = (unsigned int) strlen(fset_info->nfi_fileset) + 1; + fragment_size = LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE; + for (i = 0; i < fset_info->nfi_fragment_cnt; i++) { + if (size_remaining < LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE) + fragment_size = size_remaining; + + rc = nodemap_cluster_fileset_rec_init( + &nr_array[i], fset_info->nfi_fileset, i, fragment_size); + if (rc != 0) + GOTO(out_cleanup, rc); + + nodemap_cluster_key_init(&nk_array[i], fset_info->nfi_nm_id, + fset_info->nfi_subid + i); + size_remaining -= fragment_size; + } + rc = nodemap_idx_insert_batch(env, idx, nk_array, nr_array, + fset_info->nfi_fragment_cnt, + inserted_out); + +out_cleanup: + OBD_FREE_PTR_ARRAY(nr_array, fset_info->nfi_fragment_cnt); + OBD_FREE_PTR_ARRAY(nk_array, fset_info->nfi_fragment_cnt); + + return rc; +} + +/** + * Deletes fileset fragments (identified by struct lu_nodemap_fileset_info) from + * the nodemap IAM. + * + * \param fset_info fileset info to be deleted from the nodemap IAM + * \param env execution environment + * \param idx index object to delete from + * \param deleted_out pointer to the number of records deleted. + * May be set to NULL if not needed. + * + * \retval 0 on success + * \retval -EINVAL invalid input parameters + * \retval -ENOMEM memory allocation failure + */ +static int nodemap_idx_fileset_fragments_del( + const struct lu_nodemap_fileset_info *fset_info, + const struct lu_env *env, struct dt_object *idx, int *deleted_out) +{ + struct nodemap_key *nk_array; + unsigned int i; + int rc = 0; + + if (fset_info == NULL || fset_info->nfi_fragment_cnt == 0 || + fset_info->nfi_fragment_cnt > + LUSTRE_NODEMAP_FILESET_SUBID_RANGE - 1) + RETURN(-EINVAL); + + OBD_ALLOC_PTR_ARRAY(nk_array, fset_info->nfi_fragment_cnt); + if (nk_array == NULL) + RETURN(-ENOMEM); + + /* setup fileset fragment keys to be deleted */ + for (i = 0; i < fset_info->nfi_fragment_cnt; i++) { + nodemap_cluster_key_init(&nk_array[i], fset_info->nfi_nm_id, + fset_info->nfi_subid + i); + } + + rc = nodemap_idx_delete_batch(env, idx, nk_array, + fset_info->nfi_fragment_cnt, deleted_out); + + OBD_FREE_PTR_ARRAY(nk_array, fset_info->nfi_fragment_cnt); + + return rc; +} + +/** + * Clears the full fileset sub id range from the nodemap IAM. + * + * \param nodemap nodemap where the fileset is set + * \param env execution environment + * \param idx index object to delete from + * + * \retval 0 on success (ENOENT during idx_delete is ignored) + * \retval -ENOMEM memory allocation failure + */ +static int nodemap_idx_fileset_fragments_clear(const struct lu_nodemap *nodemap, + const struct lu_env *env, + struct dt_object *idx) +{ + struct nodemap_key *nk_array; + unsigned int count, subid, i; + int rc = 0; + + count = LUSTRE_NODEMAP_FILESET_SUBID_RANGE; + + OBD_ALLOC_PTR_ARRAY(nk_array, count); + if (nk_array == NULL) + RETURN(-ENOMEM); + + /* setup fileset fragment keys to be deleted */ + for (i = 0; i < count; i++) { + subid = NODEMAP_FILESET + i; + nodemap_cluster_key_init(&nk_array[i], nodemap->nm_id, subid); + } + + rc = nodemap_idx_delete_batch(env, idx, nk_array, count, NULL); + if (rc == -ENOENT) + rc = 0; + + OBD_FREE_PTR_ARRAY(nk_array, count); + + return rc; +} + +/** + * Initializes the fileset info structure based on nodemap and fileset info. + * + * \param fset_info fileset info structure to be initialized + * \param fileset fileset name + * \param nodemap nodemap where the fileset is set + * \param subid starting subid of the fileset in the IAM + */ +static void nodemap_fileset_info_init(struct lu_nodemap_fileset_info *fset_info, + const char *fileset, + const struct lu_nodemap *nodemap, + enum nodemap_cluster_rec_subid subid) +{ + unsigned int fset_size; + + fset_info->nfi_nm_id = nodemap->nm_id; + fset_info->nfi_subid = subid; + fset_info->nfi_fileset = fileset; + + fset_size = (unsigned int)strlen(fset_info->nfi_fileset) + 1; + fset_info->nfi_fragment_cnt = + fset_size / LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE; + + if (fset_size % LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE > 0) + fset_info->nfi_fragment_cnt++; +} + +/** + * Adds a fileset to the nodemap IAM. + * + * If an error occurs during the IAM insert operation, the already inserted + * fragments are deleted. In case the latter undo operation fails, the fileset + * is subid range is cleared and -EIO is returned. + * + * \param nodemap the nodemap to insert the fileset + * \param fileset fileset name to insert + * + * \retval 0 on success + * \retval -EINVAL invalid input parameters + * \retval -EIO undo operation failed + */ +int nodemap_idx_fileset_add(const struct lu_nodemap *nodemap, + const char *fileset) +{ + struct lu_nodemap_fileset_info fset_info; + struct lu_env env; + struct dt_object *idx; + int inserted, deleted, rc2; + int rc = 0; + + ENTRY; + + if (!nodemap_mgs()) { + if (nodemap->nm_dyn) + return 0; + + rc = -EINVAL; + CERROR("%s: cannot add nodemap config to non-existing MGS: rc = %d\n", + nodemap->nm_name, rc); + RETURN(rc); + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc) + RETURN(rc); + + idx = nodemap_mgs_ncf->ncf_obj; + + /* + * NODEMAP_FILESET + 1 is static for a single fileset. + * For multiple filesets, this will be dynamically computed + * based on an incoming fileset ID. + */ + nodemap_fileset_info_init(&fset_info, fileset, nodemap, + NODEMAP_FILESET + 1); + + inserted = 0; + rc = nodemap_idx_fileset_fragments_add(&fset_info, &env, idx, + &inserted); + + if (rc < 0 && inserted != fset_info.nfi_fragment_cnt) { + if (inserted == 0) + GOTO(out, rc); + /* Only some fileset fragments were added, attempt undo */ + fset_info.nfi_fragment_cnt = inserted; + deleted = 0; + + rc2 = nodemap_idx_fileset_fragments_del(&fset_info, &env, idx, + &deleted); + if (rc2 < 0 && deleted != fset_info.nfi_fragment_cnt) { + CERROR("%s: Undo adding fileset failed. rc = %d : rc2 = %d\n", + fset_info.nfi_fileset, rc, rc2); + /* undo failed. wipe the fileset and set error code */ + rc2 = nodemap_idx_fileset_fragments_clear(nodemap, &env, + idx); + rc = -EIO; + } + } + +out: + lu_env_fini(&env); + return rc; +} + +/** + * Updates an existing fileset on the nodemap IAM. + * + * If an error occurs during the IAM operation, an undo operation is performed. + * In case the undo operation fails, the fileset is subid range is cleared + * and -EIO is returned. + * + * \param nodemap the nodemap to update the fileset + * \param fileset_old fileset name to be deleted + * \param fileset_new fileset name to be inserted + * + * \retval 0 on success + * \retval -EINVAL invalid input parameters + * \retval -EIO undo operation failed + */ +int nodemap_idx_fileset_update(const struct lu_nodemap *nodemap, + const char *fileset_old, const char *fileset_new) +{ + struct lu_env env; + int rc = 0; + + ENTRY; + + if (!nodemap_mgs()) { + if (nodemap->nm_dyn) + return 0; + + rc = -EINVAL; + CERROR("%s: cannot add nodemap config to non-existing MGS: rc = %d\n", + nodemap->nm_name, rc); + RETURN(rc); + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + + rc = nodemap_idx_fileset_del(nodemap, fileset_old); + if (rc < 0) + GOTO(out, rc); + + rc = nodemap_idx_fileset_add(nodemap, fileset_new); + +out: + lu_env_fini(&env); + return rc; +} + +/** + * Deletes a fileset from the nodemap IAM. + * + * If an error occurs during the IAM deleted operation, the already deleted + * fragments are re-inserted. In case the latter undo operation fails, + * the fileset is subid range is cleared and -EIO is returned. + * + * \param nodemap the nodemap to delete from + * \param fileset fileset name to be deleted + * + * \retval 0 on success + * \retval -EINVAL invalid input parameters + * \retval -EIO undo operation failed + */ +int nodemap_idx_fileset_del(const struct lu_nodemap *nodemap, + const char *fileset) +{ + struct lu_env env; + struct dt_object *idx; + struct lu_nodemap_fileset_info fset_info; + int deleted, inserted, rc2; + int rc = 0; + + ENTRY; + + if (!nodemap_mgs()) { + if (nodemap->nm_dyn) + return 0; + + rc = -EINVAL; + CERROR("%s: cannot add nodemap config to non-existing MGS: rc = %d\n", + nodemap->nm_name, rc); + RETURN(rc); + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + + idx = nodemap_mgs_ncf->ncf_obj; + + nodemap_fileset_info_init(&fset_info, fileset, nodemap, + NODEMAP_FILESET + 1); + + deleted = 0; + rc = nodemap_idx_fileset_fragments_del( + &fset_info, &env, nodemap_mgs_ncf->ncf_obj, &deleted); + if (rc < 0 && deleted != fset_info.nfi_fragment_cnt) { + if (deleted == 0) + GOTO(out, rc); + /* + * Only some fileset fragments were deleted, + * attempt undo based on the initial fileset, set in fset_info + */ + fset_info.nfi_fragment_cnt = deleted; + rc2 = nodemap_idx_fileset_fragments_add(&fset_info, &env, idx, + &inserted); + if (rc2 < 0 && inserted != fset_info.nfi_fragment_cnt) { + CERROR("%s: Undo deleting fileset failed. rc = %d : rc2 = %d\n", + fset_info.nfi_fileset, rc, rc2); + /* undo failed. wipe the fileset and set error code */ + rc2 = nodemap_idx_fileset_fragments_clear(nodemap, &env, + idx); + rc = -EIO; + } + } + +out: + lu_env_fini(&env); + return rc; +} + +/** + * Clears a fileset subid range from the nodemap IAM. + * + * + * \param nodemap nodemap where the fileset is set to be cleared + * + * \retval 0 on success (ENOENT during idx_delete is ignored) + * \retval -EINVAL invalid input parameters + */ +int nodemap_idx_fileset_clear(const struct lu_nodemap *nodemap) +{ + struct lu_env env; + struct dt_object *idx; + int rc = 0; + + ENTRY; + + if (!nodemap_mgs()) { + if (nodemap->nm_dyn) + return 0; + + rc = -EINVAL; + CERROR("%s: cannot add nodemap config to non-existing MGS: rc = %d\n", + nodemap->nm_name, rc); + RETURN(rc); + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + + idx = nodemap_mgs_ncf->ncf_obj; + + rc = nodemap_idx_fileset_fragments_clear(nodemap, &env, idx); + + lu_env_fini(&env); + return rc; +} + int nodemap_idx_range_add(const struct lu_nid_range *range) { struct nodemap_key nk; @@ -865,20 +1370,27 @@ static int nodemap_cluster_rec_helper(struct nodemap_config *config, flags2 = rec->ncr.ncr_flags2; nodemap->nmf_readonly_mount = flags2 & NM_FL2_READONLY_MOUNT; nodemap->nmf_deny_mount = flags2 & NM_FL2_DENY_MOUNT; + nodemap->nmf_fileset_use_iam = flags2 & NM_FL2_FILESET_USE_IAM; + /* by default, and in the absence of cluster_roles, grant all roles */ nodemap->nmf_rbac = NODEMAP_RBAC_ALL; - /* The fileset should be saved otherwise it will be empty - * every time in case of "NODEMAP_CLUSTER_IDX". + /* + * If the use IAM flag has not been set on the nodemap, a llog-based + * fileset may be in use on the old nodemap. It needs to be separately + * copied to the new nodemap as it is otherwise lost when the IAM + * is read for type "NODEMAP_CLUSTER_IDX". */ - mutex_lock(&active_config_lock); - old_nm = nodemap_lookup(rec->ncr.ncr_name); - if (!IS_ERR(old_nm) && old_nm->nm_fileset[0] != '\0') - strscpy(nodemap->nm_fileset, old_nm->nm_fileset, - sizeof(nodemap->nm_fileset)); - mutex_unlock(&active_config_lock); - if (!IS_ERR(old_nm)) - nodemap_putref(old_nm); + if (!nodemap->nmf_fileset_use_iam) { + mutex_lock(&active_config_lock); + old_nm = nodemap_lookup(rec->ncr.ncr_name); + if (!IS_ERR(old_nm) && old_nm->nm_fileset[0] != '\0') + strscpy(nodemap->nm_fileset, old_nm->nm_fileset, + sizeof(nodemap->nm_fileset)); + mutex_unlock(&active_config_lock); + if (!IS_ERR(old_nm)) + nodemap_putref(old_nm); + } if (*recent_nodemap == NULL) { *recent_nodemap = nodemap; @@ -900,6 +1412,37 @@ static int nodemap_cluster_roles_helper(struct lu_nodemap *nodemap, } /** + * Process a fileset fragment and apply it to the current nodemap. The incoming + * path fragment is copied to the nodemap fileset based on the fragment ID + * which is used to compute the char* offset. + * + * Fragments processed by this function do not need to be in order. + * + * \param nodemap nodemap to update with this fileset fragment + * \param rec fileset fragment record + */ +static int nodemap_cluster_fileset_helper(struct lu_nodemap *nodemap, + const union nodemap_rec *rec) +{ + unsigned int fragment_id, fragment_len, fset_offset, fset_len_remain; + + fragment_id = le16_to_cpu(rec->nfr.nfr_fragment_id); + fragment_len = LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE; + + /* compute nodemap fileset position */ + fset_offset = fragment_id * LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE; + fset_len_remain = sizeof(nodemap->nm_fileset) - fset_offset; + + if (fragment_len > fset_len_remain) + fragment_len = fset_len_remain; + + memcpy(nodemap->nm_fileset + fset_offset, rec->nfr.nfr_path_fragment, + fragment_len); + + return 0; +} + +/** * Process a key/rec pair and modify the new configuration. * * \param config configuration to update with this key/rec data @@ -921,10 +1464,10 @@ static int nodemap_process_keyrec(struct nodemap_config *config, enum nodemap_idx_type type; enum nodemap_id_type id_type; struct lnet_nid nid[2]; - int subtype; + int subtype, cluster_idx_key; u32 nodemap_id; u32 map[2]; - int rc; + int rc = 0; ENTRY; @@ -961,7 +1504,7 @@ static int nodemap_process_keyrec(struct nodemap_config *config, GOTO(out, rc = -ENOENT); } - /* update most recently used nodemap if necessay */ + /* update most recently used nodemap if necessary */ if (nodemap != *recent_nodemap) *recent_nodemap = nodemap; } @@ -973,33 +1516,32 @@ static int nodemap_process_keyrec(struct nodemap_config *config, nodemap->nm_name, nodemap_id); break; case NODEMAP_CLUSTER_IDX: - switch (nodemap_get_key_subtype(key)) { - case NODEMAP_CLUSTER_REC: + { + cluster_idx_key = nodemap_get_key_subtype(key); + if (cluster_idx_key == NODEMAP_CLUSTER_REC) { rc = nodemap_cluster_rec_helper(config, nodemap_id, rec, recent_nodemap); - if (rc != 0) - GOTO(out, rc); - break; - case NODEMAP_CLUSTER_ROLES: + } else if (cluster_idx_key == NODEMAP_CLUSTER_ROLES) { rc = nodemap_cluster_roles_helper(nodemap, rec); - if (rc != 0) - GOTO(out, rc); - break; - case NODEMAP_CLUSTER_OFFSET: + } else if (cluster_idx_key == NODEMAP_CLUSTER_OFFSET) { /* only works for offset UID = GID = PROJID */ - rc = nodemap_add_offset_helper(nodemap, - le32_to_cpu(rec->nor.nor_start_uid), - le32_to_cpu(rec->nor.nor_limit_uid)); - if (rc != 0) - GOTO(out, rc); - break; - default: + rc = nodemap_add_offset_helper( + nodemap, le32_to_cpu(rec->nor.nor_start_uid), + le32_to_cpu(rec->nor.nor_limit_uid)); + } else if (cluster_idx_key >= NODEMAP_FILESET && + cluster_idx_key < + NODEMAP_FILESET + + LUSTRE_NODEMAP_FILESET_SUBID_RANGE) { + rc = nodemap_cluster_fileset_helper(nodemap, rec); + } else { CWARN("%s: ignoring keyrec of type %d with subtype %u\n", nodemap->nm_name, NODEMAP_CLUSTER_IDX, nodemap_get_key_subtype(key)); - break; } + if (rc != 0) + GOTO(out, rc); break; + } case NODEMAP_RANGE_IDX: lnet_nid4_to_nid(le64_to_cpu(rec->nrr.nrr_start_nid), &nid[0]); lnet_nid4_to_nid(le64_to_cpu(rec->nrr.nrr_end_nid), &nid[1]); diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index da50441..f5510e4 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -6418,6 +6418,23 @@ void lustre_assert_wire_constants(void) LASSERTF((int)sizeof(((struct nodemap_offset_rec *)0)->nor_padding2) == 4, "found %lld\n", (long long)(int)sizeof(((struct nodemap_offset_rec *)0)->nor_padding2)); + /* Checks for struct nodemap_fileset_rec */ + LASSERTF((int)sizeof(struct nodemap_fileset_rec) == 32, "found %lld\n", + (long long)(int)sizeof(struct nodemap_fileset_rec)); + BUILD_BUG_ON(LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE != 28); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_path_fragment[28]) == 28, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_path_fragment[28])); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_path_fragment[28]) == 1, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_path_fragment[28])); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_fragment_id) == 28, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_fragment_id)); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_fragment_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_fragment_id)); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_padding1) == 30, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_padding1)); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_padding1) == 2, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_padding1)); + /* Checks for union nodemap_rec */ LASSERTF((int)sizeof(union nodemap_rec) == 32, "found %lld\n", (long long)(int)sizeof(union nodemap_rec)); @@ -6489,6 +6506,8 @@ void lustre_assert_wire_constants(void) (unsigned)NM_FL2_READONLY_MOUNT); LASSERTF(NM_FL2_DENY_MOUNT == 0x00000002UL, "found 0x%.8xUL\n", (unsigned int)NM_FL2_DENY_MOUNT); + LASSERTF(NM_FL2_FILESET_USE_IAM == 0x00000004UL, "found 0x%.8xUL\n", + (unsigned)NM_FL2_FILESET_USE_IAM); LASSERTF(NODEMAP_UID == 0, "found %lld\n", (long long)NODEMAP_UID); LASSERTF(NODEMAP_GID == 1, "found %lld\n", diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 2f6974f..c8f404d 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2137,22 +2137,36 @@ test_26() { } run_test 26 "test transferring very large nodemap" +nodemap_exercise_fileset_cleanup() { + # Already mounted clients are skipped in zconf_mount_clients() + for client in "${clients_arr[@]}"; do + zconf_mount_clients $client $MOUNT $MOUNT_OPTS || + error "unable to mount client $client" + done +} + nodemap_exercise_fileset() { - local nm="$1" - local loop=0 + local have_persistent_fset_cmd local check_proj=true + local loop=0 + local nm="$1" (( $MDS1_VERSION >= $(version_code 2.14.52) )) || check_proj=false + # when "have_persistent_fset_cmd" is true, "lctl nodemap_set_fileset" + # is persistent, otherwise "lctl set_param -P" must be used + have_persistent_fset_cmd=false + (( $MGS_VERSION >= $(version_code 2.16.51) )) && + have_persistent_fset_cmd=true + # setup - if [ "$nm" == "default" ]; then + if [[ "$nm" == "default" ]]; then do_facet mgs $LCTL nodemap_activate 1 wait_nm_sync active do_facet mgs $LCTL nodemap_modify --name default \ --property admin --value 1 do_facet mgs $LCTL nodemap_modify --name default \ --property trusted --value 1 - wait_nm_sync default admin_nodemap wait_nm_sync default trusted_nodemap check_proj=false else @@ -2167,13 +2181,27 @@ nodemap_exercise_fileset() { fileset_test_setup "$nm" # add fileset info to $nm nodemap - if ! combined_mgs_mds; then - do_facet mgs $LCTL set_param nodemap.${nm}.fileset=/$subdir || - error "unable to add fileset info to $nm nodemap on MGS" + if $have_persistent_fset_cmd; then + do_facet mgs $LCTL nodemap_set_fileset --name $nm \ + --fileset "/${subdir}" || + error "can't set fileset to $nm nodemap on MGS" + # check fileset is set on local mgs node + wait_update_facet mgs "$LCTL get_param nodemap.${nm}.fileset" \ + "nodemap.${nm}.fileset=/${subdir}" || + error "fileset /${subdir} not set on $nm nodemap" + else + if ! combined_mgs_mds; then + do_facet mgs $LCTL set_param \ + nodemap.${nm}.fileset=/${subdir} || + error "can't set fileset /${subdir} to $nm nodemap on MGS" + fi + do_facet mgs $LCTL set_param -P \ + nodemap.${nm}.fileset=/${subdir} || + error "can't set fileset /${subdir} to $nm nodemap on servers" fi - do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=/$subdir || - error "unable to add fileset info to $nm nodemap for servers" - wait_nm_sync $nm fileset "nodemap.${nm}.fileset=/$subdir" + + # check fileset is set on remote nodes + wait_nm_sync $nm fileset "nodemap.${nm}.fileset=/${subdir}" if $check_proj; then do_facet mgs $LCTL nodemap_modify --name $nm \ @@ -2192,9 +2220,21 @@ nodemap_exercise_fileset() { wait_nm_sync $nm deny_unknown fi - # re-mount client - zconf_umount_clients ${clients_arr[0]} $MOUNT || - error "unable to umount client ${clients_arr[0]}" + # re-start all components to verify persistence of fileset after restart + stopall || error "unable to stop" + # Unload modules to fully reload nodemap IAM + LOAD_MODULES_REMOTE=true unload_modules || + error "unable to unload modules" + LOAD_MODULES_REMOTE=true load_modules || + error "unable to load modules" + mountmgs || error "unable to start mgs" + mountmds || error "unable to start mds" + mountoss || error "unable to start oss" + + stack_trap nodemap_exercise_fileset_cleanup EXIT + + # mount a single client for fileset testing and remount + # the remaining clients later. # set some generic fileset to trigger SSK code export FILESET=/ zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS || @@ -2227,11 +2267,16 @@ nodemap_exercise_fileset() { # remove fileset info from nodemap do_facet mgs $LCTL nodemap_set_fileset --name $nm --fileset clear || error "unable to delete fileset info on $nm nodemap" + # check whether fileset was removed on mgs wait_update_facet mgs "$LCTL get_param nodemap.${nm}.fileset" \ "nodemap.${nm}.fileset=" || error "fileset info still not cleared on $nm nodemap" - do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=clear || + if ! $have_persistent_fset_cmd; then + do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=clear || error "unable to reset fileset info on $nm nodemap" + fi + + # check whether fileset was removed on remote nodes wait_nm_sync $nm fileset "nodemap.${nm}.fileset=" do_facet mgs $LCTL set_param -P -d nodemap.${nm}.fileset || error "unable to remove fileset rule on $nm nodemap" @@ -2255,12 +2300,11 @@ nodemap_exercise_fileset() { error "unable to umount client ${clients_arr[0]}" fi fileset_test_cleanup "$nm" - if [ "$nm" == "default" ]; then + if [[ "$nm" == "default" ]]; then do_facet mgs $LCTL nodemap_modify --name default \ --property admin --value 0 do_facet mgs $LCTL nodemap_modify --name default \ --property trusted --value 0 - wait_nm_sync default admin_nodemap wait_nm_sync default trusted_nodemap do_facet mgs $LCTL nodemap_activate 0 wait_nm_sync active 0 @@ -2269,30 +2313,29 @@ nodemap_exercise_fileset() { else nodemap_test_cleanup fi - if $SHARED_KEY; then - zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS || - error "unable to remount client ${clients_arr[0]}" - fi + # The fileset cleanup trap is reset during nodemap clean up. + # Call fileset cleanup to restart all shut down clients + nodemap_exercise_fileset_cleanup } test_27a() { - [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] && + (( $MDS1_VERSION < $(version_code 2.11.50) )) && skip "Need MDS >= 2.11.50" # if servers run on the same node, it is impossible to tell if they get # synced with the mgs, so this test needs to be skipped - if [ $(facet_active_host mgs) == $(facet_active_host mds) ] && - [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then + if [[ $(facet_active_host mgs) == $(facet_active_host mds) ]] && + [[ $(facet_active_host mgs) == $(facet_active_host ost1) ]]; then skip "local mode not supported" fi for nm in "default" "c0"; do - local subdir="subdir_${nm}" + local subdir="thisisaverylongsubdirtotestlongfilesetsandtotestmultiplefilesetfragmentsonthenodemapiam_${nm}" local subsubdir="subsubdir_${nm}" - if [ "$nm" == "default" ] && [ "$SHARED_KEY" == "true" ]; then - echo "Skipping nodemap $nm with SHARED_KEY"; - continue; + if [[ "$nm" == "default" && "$SHARED_KEY" == "true" ]]; then + echo "Skipping nodemap $nm with SHARED_KEY" + continue fi echo "Exercising fileset for nodemap $nm" diff --git a/lustre/utils/lustre_param.c b/lustre/utils/lustre_param.c index 5143d69..8b35e8a 100644 --- a/lustre/utils/lustre_param.c +++ b/lustre/utils/lustre_param.c @@ -61,6 +61,7 @@ #include "lctl_thread.h" #include "lustreapi_internal.h" +#include "lstddef.h" #include #include @@ -868,6 +869,46 @@ int jt_lcfg_getparam(int argc, char **argv) } /** + * Parses a cleaned set_param path and checks whether it is deprecated. If yes, + * the user is notified with a warning. This function does not exit the program. + * + * \param[in] path The set_param key to be checked for deprecation. + * + */ +static void setparam_check_deprecated(const char *path) +{ + regex_t regex; + int err, i; + + struct deprecated_param { + const char *regex; + const char *message; + }; + + static const struct deprecated_param deprecated_params[] = { + { .regex = "^nodemap/[^/]+/fileset$", + .message = + "Warning: The parameter '%s' is deprecated. Please use \"lctl nodemap_set_fileset\" instead.\n" }, + /* Add more deprecated parameters here in the future */ + }; + + for (i = 0; i < ARRAY_SIZE(deprecated_params); i++) { + err = regcomp(®ex, deprecated_params[i].regex, REG_EXTENDED); + if (err) { + fprintf(stderr, "Error compiling regex: %s\n", + deprecated_params[i].regex); + continue; + } + + err = regexec(®ex, path, 0, NULL, 0); + if (!err) + fprintf(stdout, deprecated_params[i].message, path); + + regfree(®ex); + } +} + +/** * Parses the commandline options to set_param. * * \param[in] argc count of arguments given to set_param @@ -1016,6 +1057,8 @@ int jt_lcfg_setparam(int argc, char **argv) if (rc < 0) break; + setparam_check_deprecated(path); + rc = do_param_op(&popt, path, value, SET_PARAM, wq_ptr); if (rc < 0) { if (rc == -ENOENT && getuid() != 0) { diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 8690ef4..2a4d85e 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -3021,6 +3021,16 @@ static void check_nodemap_cluster_roles_rec(void) CHECK_MEMBER(nodemap_cluster_roles_rec, ncrr_padding3); } +static void check_nodemap_fileset_rec(void) +{ + BLANK_LINE(); + CHECK_STRUCT(nodemap_fileset_rec); + CHECK_CDEFINE(LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE); + CHECK_MEMBER(nodemap_fileset_rec, nfr_path_fragment[LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE]); + CHECK_MEMBER(nodemap_fileset_rec, nfr_fragment_id); + CHECK_MEMBER(nodemap_fileset_rec, nfr_padding1); +} + static void check_nodemap_rec(void) { BLANK_LINE(); @@ -3062,6 +3072,7 @@ static void check_nodemap_key(void) CHECK_VALUE_X(NM_FL_MAP_PROJID); CHECK_VALUE_X(NM_FL2_READONLY_MOUNT); CHECK_VALUE_X(NM_FL2_DENY_MOUNT); + CHECK_VALUE_X(NM_FL2_FILESET_USE_IAM); CHECK_VALUE(NODEMAP_UID); CHECK_VALUE(NODEMAP_GID); @@ -3803,6 +3814,7 @@ main(int argc, char **argv) check_nodemap_offset_rec(); check_nodemap_global_rec(); check_nodemap_cluster_roles_rec(); + check_nodemap_fileset_rec(); check_nodemap_rec(); check_nodemap_key(); diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index 9e4819f..1d57383 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -6461,6 +6461,23 @@ void lustre_assert_wire_constants(void) LASSERTF((int)sizeof(((struct nodemap_offset_rec *)0)->nor_padding2) == 4, "found %lld\n", (long long)(int)sizeof(((struct nodemap_offset_rec *)0)->nor_padding2)); + /* Checks for struct nodemap_fileset_rec */ + LASSERTF((int)sizeof(struct nodemap_fileset_rec) == 32, "found %lld\n", + (long long)(int)sizeof(struct nodemap_fileset_rec)); + BUILD_BUG_ON(LUSTRE_NODEMAP_FILESET_FRAGMENT_SIZE != 28); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_path_fragment[28]) == 28, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_path_fragment[28])); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_path_fragment[28]) == 1, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_path_fragment[28])); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_fragment_id) == 28, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_fragment_id)); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_fragment_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_fragment_id)); + LASSERTF((int)offsetof(struct nodemap_fileset_rec, nfr_padding1) == 30, "found %lld\n", + (long long)(int)offsetof(struct nodemap_fileset_rec, nfr_padding1)); + LASSERTF((int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_padding1) == 2, "found %lld\n", + (long long)(int)sizeof(((struct nodemap_fileset_rec *)0)->nfr_padding1)); + /* Checks for union nodemap_rec */ LASSERTF((int)sizeof(union nodemap_rec) == 32, "found %lld\n", (long long)(int)sizeof(union nodemap_rec)); @@ -6532,6 +6549,8 @@ void lustre_assert_wire_constants(void) (unsigned)NM_FL2_READONLY_MOUNT); LASSERTF(NM_FL2_DENY_MOUNT == 0x00000002UL, "found 0x%.8xUL\n", (unsigned int)NM_FL2_DENY_MOUNT); + LASSERTF(NM_FL2_FILESET_USE_IAM == 0x00000004UL, "found 0x%.8xUL\n", + (unsigned)NM_FL2_FILESET_USE_IAM); LASSERTF(NODEMAP_UID == 0, "found %lld\n", (long long)NODEMAP_UID); LASSERTF(NODEMAP_GID == 1, "found %lld\n", -- 1.8.3.1