From d2629cfcabfa3a22ddf4a6c474364d0012d80390 Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Mon, 24 Jul 2017 11:58:05 -0400 Subject: [PATCH] LU-9727 lustre: add client NID to Changelogs entries Add a new changelog extension named changelog_ext_nid to hold client's NID information. NID info is added to every Changelog entry type except MARK, in the form 'nid=': 1 01CREAT 15:50:20.834838318 2017.07.24 0x0 t=[0x200000401:0x2:0x0] ef=0x3 u=500:500 nid=10.128.11.159@tcp p=[0x200000007:0x1:0x0] fileA Signed-off-by: Sebastien Buisson Change-Id: I1049a699c17d3829d38abfade3187a28ca457bd1 Reviewed-on: https://review.whamcloud.com/28213 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Quentin Bouget Reviewed-by: Li Xi Reviewed-by: Oleg Drokin --- lustre/include/md_object.h | 1 + lustre/include/uapi/linux/lustre/lustre_idl.h | 3 +- lustre/include/uapi/linux/lustre/lustre_user.h | 47 ++++++++++++++++++++++++-- lustre/mdc/mdc_changelog.c | 3 +- lustre/mdd/mdd_dir.c | 13 ++++++- lustre/mdd/mdd_internal.h | 2 ++ lustre/mdd/mdd_object.c | 3 ++ lustre/mdt/mdt_lib.c | 10 ++++++ lustre/obdclass/llog_osd.c | 44 ++++++++++-------------- lustre/utils/lfs.c | 11 +++++- lustre/utils/liblustreapi_chlg.c | 2 ++ lustre/utils/lustre_rsync.c | 3 +- 12 files changed, 109 insertions(+), 33 deletions(-) diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 845ccd05..545a827 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -675,6 +675,7 @@ struct lu_ucred { struct group_info *uc_ginfo; struct md_identity *uc_identity; char uc_jobid[LUSTRE_JOBID_SIZE]; + lnet_nid_t uc_nid; }; struct lu_ucred *lu_ucred(const struct lu_env *env); diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index aae5821..c3ec65b 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -2843,13 +2843,14 @@ enum llog_flag { LLOG_F_IS_FIXSIZE = 0x10, LLOG_F_EXT_EXTRA_FLAGS = 0x20, LLOG_F_EXT_X_UIDGID = 0x40, + LLOG_F_EXT_X_NID = 0x80, /* Note: Flags covered by LLOG_F_EXT_MASK will be inherited from * catlog to plain log, so do not add LLOG_F_IS_FIXSIZE here, * because the catlog record is usually fixed size, but its plain * log record can be variable */ LLOG_F_EXT_MASK = LLOG_F_EXT_JOBID | LLOG_F_EXT_EXTRA_FLAGS | - LLOG_F_EXT_X_UIDGID, + LLOG_F_EXT_X_UIDGID | LLOG_F_EXT_X_NID, }; /* On-disk header structure of each log object, stored in little endian order */ diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 09c139e..0fdc9d1 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -1121,7 +1121,8 @@ static inline void hsm_set_cl_error(int *flags, int error) enum changelog_rec_extra_flags { CLFE_INVALID = 0, CLFE_UIDGID = 0x0001, - CLFE_SUPPORTED = CLFE_UIDGID + CLFE_NID = 0x0002, + CLFE_SUPPORTED = CLFE_UIDGID | CLFE_NID }; enum changelog_send_flag { @@ -1140,6 +1141,8 @@ enum changelog_send_flag { enum changelog_send_extra_flag { /* Pack uid/gid into the changelog record */ CHANGELOG_EXTRA_FLAG_UIDGID = 0x01, + /* Pack nid into the changelog record */ + CHANGELOG_EXTRA_FLAG_NID = 0x02, }; #define CR_MAXSIZE cfs_size_round(2 * NAME_MAX + 2 + \ @@ -1192,6 +1195,15 @@ struct changelog_ext_uidgid { __u64 cr_gid; }; +/* Changelog extra extension to include NID. */ +struct changelog_ext_nid { + /* have __u64 instead of lnet_nid_t type for use by client api */ + __u64 cr_nid; + /* for use when IPv6 support is added */ + __u64 extra; + __u32 padding; +}; + static inline struct changelog_ext_extra_flags *changelog_rec_extra_flags( const struct changelog_rec *rec); @@ -1210,6 +1222,8 @@ static inline size_t changelog_rec_offset(enum changelog_rec_flags crf, size += sizeof(struct changelog_ext_extra_flags); if (cref & CLFE_UIDGID) size += sizeof(struct changelog_ext_uidgid); + if (cref & CLFE_NID) + size += sizeof(struct changelog_ext_nid); } return size; @@ -1278,6 +1292,22 @@ struct changelog_ext_uidgid *changelog_rec_uidgid( CLFE_INVALID)); } +/* The nid is the second extra extension */ +static inline +struct changelog_ext_nid *changelog_rec_nid(const struct changelog_rec *rec) +{ + enum changelog_rec_flags crf = rec->cr_flags & + (CLF_VERSION | CLF_RENAME | CLF_JOBID | CLF_EXTRA_FLAGS); + enum changelog_rec_extra_flags cref = CLFE_INVALID; + + if (rec->cr_flags & CLF_EXTRA_FLAGS) + cref = changelog_rec_extra_flags(rec)->cr_extra_flags & + CLFE_UIDGID; + + return (struct changelog_ext_nid *)((char *)rec + + changelog_rec_offset(crf, cref)); +} + /* The name follows the rename, jobid and extra flags extns, if present */ static inline char *changelog_rec_name(const struct changelog_rec *rec) { @@ -1327,6 +1357,7 @@ static inline void changelog_remap_rec(struct changelog_rec *rec, enum changelog_rec_flags crf_wanted, enum changelog_rec_extra_flags cref_want) { + char *nid_mov = NULL; char *uidgid_mov = NULL; char *ef_mov; char *jid_mov; @@ -1351,9 +1382,13 @@ static inline void changelog_remap_rec(struct changelog_rec *rec, /* Locations of extensions in the remapped record */ if (rec->cr_flags & CLF_EXTRA_FLAGS) { + nid_mov = (char *)rec + + changelog_rec_offset(crf_wanted & CLF_SUPPORTED, + cref_want & ~CLFE_NID); uidgid_mov = (char *)rec + changelog_rec_offset(crf_wanted & CLF_SUPPORTED, - CLFE_INVALID); + cref_want & ~(CLFE_UIDGID | + CLFE_NID)); cref = changelog_rec_extra_flags(rec)->cr_extra_flags; } @@ -1374,6 +1409,10 @@ static inline void changelog_remap_rec(struct changelog_rec *rec, /* Move the extension fields to the desired positions */ if ((crf_wanted & CLF_EXTRA_FLAGS) && (rec->cr_flags & CLF_EXTRA_FLAGS)) { + if ((cref_want & CLFE_NID) && (cref & CLFE_NID)) + memmove(nid_mov, changelog_rec_nid(rec), + sizeof(struct changelog_ext_nid)); + if ((cref_want & CLFE_UIDGID) && (cref & CLFE_UIDGID)) memmove(uidgid_mov, changelog_rec_uidgid(rec), sizeof(struct changelog_ext_uidgid)); @@ -1391,6 +1430,10 @@ static inline void changelog_remap_rec(struct changelog_rec *rec, sizeof(struct changelog_ext_rename)); /* Clear newly added fields */ + if (nid_mov && (cref_want & CLFE_NID) && + !(cref & CLFE_NID)) + memset(nid_mov, 0, sizeof(struct changelog_ext_nid)); + if (uidgid_mov && (cref_want & CLFE_UIDGID) && !(cref & CLFE_UIDGID)) memset(uidgid_mov, 0, sizeof(struct changelog_ext_uidgid)); diff --git a/lustre/mdc/mdc_changelog.c b/lustre/mdc/mdc_changelog.c index 7a585d3..252592c 100644 --- a/lustre/mdc/mdc_changelog.c +++ b/lustre/mdc/mdc_changelog.c @@ -220,7 +220,8 @@ static int chlg_load(void *args) LLOG_F_IS_CAT | LLOG_F_EXT_JOBID | LLOG_F_EXT_EXTRA_FLAGS | - LLOG_F_EXT_X_UIDGID, + LLOG_F_EXT_X_UIDGID | + LLOG_F_EXT_X_NID, NULL); if (rc) { CERROR("%s: fail to init llog handle: rc = %d\n", diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index af74cfe..09d41d6 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -738,7 +738,7 @@ static int mdd_llog_record_calc_size(const struct lu_env *env, { const struct lu_ucred *uc = lu_ucred(env); enum changelog_rec_flags crf = CLF_EXTRA_FLAGS; - enum changelog_rec_extra_flags crfe = CLFE_UIDGID; + enum changelog_rec_extra_flags crfe = CLFE_UIDGID | CLFE_NID; if (sname != NULL) crf |= CLF_RENAME; @@ -1054,6 +1054,14 @@ void mdd_changelog_rec_extra_uidgid(struct changelog_rec *rec, uidgid->cr_gid = gid; } +void mdd_changelog_rec_extra_nid(struct changelog_rec *rec, + lnet_nid_t nid) +{ + struct changelog_ext_nid *clnid = changelog_rec_nid(rec); + + clnid->cr_nid = nid; +} + /** Store a namespace change changelog record * If this fails, we must fail the whole transaction; we don't * want the change to commit without the log entry. @@ -1109,6 +1117,7 @@ int mdd_changelog_ns_store(const struct lu_env *env, if (uc->uc_jobid[0] != '\0') crf |= CLF_JOBID; xflags |= CLFE_UIDGID; + xflags |= CLFE_NID; } if (sname != NULL) @@ -1123,6 +1132,8 @@ int mdd_changelog_ns_store(const struct lu_env *env, if (xflags & CLFE_UIDGID) mdd_changelog_rec_extra_uidgid(&rec->cr, uc->uc_uid, uc->uc_gid); + if (xflags & CLFE_NID) + mdd_changelog_rec_extra_nid(&rec->cr, uc->uc_nid); } rec->cr.cr_type = (__u32)type; diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index 425dc14..bc27a30 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -308,6 +308,8 @@ void mdd_changelog_rec_ext_jobid(struct changelog_rec *rec, const char *jobid); void mdd_changelog_rec_ext_extra_flags(struct changelog_rec *rec, __u64 eflags); void mdd_changelog_rec_extra_uidgid(struct changelog_rec *rec, __u64 uid, __u64 gid); +void mdd_changelog_rec_extra_nid(struct changelog_rec *rec, + lnet_nid_t nid); int mdd_changelog_store(const struct lu_env *env, struct mdd_device *mdd, struct llog_changelog_rec *rec, struct thandle *th); int mdd_changelog_data_store(const struct lu_env *env, struct mdd_device *mdd, diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index 280a756..05b67e4 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -652,6 +652,7 @@ static int mdd_changelog_data_store_by_fid(const struct lu_env *env, if (uc->uc_jobid[0] != '\0') flags |= CLF_JOBID; xflags |= CLFE_UIDGID; + xflags |= CLFE_NID; } reclen = llog_data_len(LLOG_CHANGELOG_HDR_SZ + @@ -676,6 +677,8 @@ static int mdd_changelog_data_store_by_fid(const struct lu_env *env, if (xflags & CLFE_UIDGID) mdd_changelog_rec_extra_uidgid(&rec->cr, uc->uc_uid, uc->uc_gid); + if (xflags & CLFE_NID) + mdd_changelog_rec_extra_nid(&rec->cr, uc->uc_nid); } rc = mdd_changelog_store(env, mdd, rec, handle); diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index 21b0d42..70935d0 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -141,6 +141,14 @@ static void ucred_set_jobid(struct mdt_thread_info *info, struct lu_ucred *uc) uc->uc_jobid[0] = '\0'; } +static void ucred_set_nid(struct mdt_thread_info *info, struct lu_ucred *uc) +{ + if (info && info->mti_exp && info->mti_exp->exp_connection) + uc->uc_nid = info->mti_exp->exp_connection->c_peer.nid; + else + uc->uc_nid = LNET_NID_ANY; +} + static int new_init_ucred(struct mdt_thread_info *info, ucred_init_type_t type, void *buf, bool drop_fs_cap) { @@ -299,6 +307,7 @@ static int new_init_ucred(struct mdt_thread_info *info, ucred_init_type_t type, ucred->uc_cap = pud->pud_cap; ucred->uc_valid = UCRED_NEW; ucred_set_jobid(info, ucred); + ucred_set_nid(info, ucred); EXIT; @@ -471,6 +480,7 @@ static int old_init_ucred_common(struct mdt_thread_info *info, uc->uc_cap &= ~CFS_CAP_FS_MASK; uc->uc_valid = UCRED_OLD; ucred_set_jobid(info, uc); + ucred_set_nid(info, uc); EXIT; diff --git a/lustre/obdclass/llog_osd.c b/lustre/obdclass/llog_osd.c index 7035171..c853433 100644 --- a/lustre/obdclass/llog_osd.c +++ b/lustre/obdclass/llog_osd.c @@ -788,9 +788,23 @@ static inline void llog_skip_over(struct llog_handle *lgh, __u64 *off, */ static void changelog_block_trim_ext(struct llog_rec_hdr *hdr, struct llog_rec_hdr *last_hdr, - enum changelog_rec_flags flags, - enum changelog_rec_extra_flags extra_flags) + struct llog_handle *loghandle) { + enum changelog_rec_flags flags = CLF_SUPPORTED; + enum changelog_rec_extra_flags extra_flags = CLFE_SUPPORTED; + + if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_NID)) + extra_flags &= ~CLFE_NID; + if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_UIDGID)) + extra_flags &= ~CLFE_UIDGID; + if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_EXTRA_FLAGS)) + flags &= ~CLF_EXTRA_FLAGS; + if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID)) + flags &= ~CLF_JOBID; + + if (flags == CLF_SUPPORTED && extra_flags == CLFE_SUPPORTED) + return; + if (hdr->lrh_type != CHANGELOG_REC) return; @@ -840,8 +854,6 @@ static int llog_osd_next_block(const struct lu_env *env, int last_idx = *cur_idx; __u64 last_offset = *cur_offset; bool force_mini_rec = false; - enum changelog_rec_flags flags; - enum changelog_rec_extra_flags xflags; ENTRY; @@ -973,16 +985,7 @@ static int llog_osd_next_block(const struct lu_env *env, } /* Trim unsupported extensions for compat w/ older clients */ - flags = CLF_SUPPORTED; - xflags = CLFE_SUPPORTED; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_UIDGID)) - xflags &= ~CLFE_UIDGID; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_EXTRA_FLAGS)) - flags &= ~CLF_EXTRA_FLAGS; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID)) - flags &= ~CLF_JOBID; - if (flags != CLF_SUPPORTED || xflags != CLFE_SUPPORTED) - changelog_block_trim_ext(rec, last_rec, flags, xflags); + changelog_block_trim_ext(rec, last_rec, loghandle); GOTO(out, rc = 0); @@ -1025,8 +1028,6 @@ static int llog_osd_prev_block(const struct lu_env *env, struct dt_device *dt; loff_t cur_offset; __u32 chunk_size; - enum changelog_rec_flags flags; - enum changelog_rec_extra_flags xflags; int rc; ENTRY; @@ -1119,16 +1120,7 @@ static int llog_osd_prev_block(const struct lu_env *env, } /* Trim unsupported extensions for compat w/ older clients */ - flags = CLF_SUPPORTED; - xflags = CLFE_SUPPORTED; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_UIDGID)) - xflags &= ~CLFE_UIDGID; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_EXTRA_FLAGS)) - flags &= ~CLF_EXTRA_FLAGS; - if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID)) - flags &= ~CLF_JOBID; - if (flags != CLF_SUPPORTED || xflags != CLFE_SUPPORTED) - changelog_block_trim_ext(rec, last_rec, flags, xflags); + changelog_block_trim_ext(rec, last_rec, loghandle); GOTO(out, rc = 0); } diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index a957a2a..98d78e4 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -71,6 +71,7 @@ #include #include #include +#include #ifndef ARRAY_SIZE # define ARRAY_SIZE(a) ((sizeof(a)) / (sizeof((a)[0]))) @@ -5601,7 +5602,8 @@ static int lfs_changelog(int argc, char **argv) } rc = llapi_changelog_set_xflags(changelog_priv, - CHANGELOG_EXTRA_FLAG_UIDGID); + CHANGELOG_EXTRA_FLAG_UIDGID | + CHANGELOG_EXTRA_FLAG_NID); if (rc < 0) { fprintf(stderr, "Can't set xflags for changelog: %s\n", strerror(errno = -rc)); @@ -5652,6 +5654,13 @@ static int lfs_changelog(int argc, char **argv) printf(" u=%llu:%llu", uidgid->cr_uid, uidgid->cr_gid); } + if (ef->cr_extra_flags & CLFE_NID) { + struct changelog_ext_nid *nid = + changelog_rec_nid(rec); + + printf(" nid=%s", + libcfs_nid2str(nid->cr_nid)); + } } if (rec->cr_namelen) diff --git a/lustre/utils/liblustreapi_chlg.c b/lustre/utils/liblustreapi_chlg.c index 1389998..32a31c2 100644 --- a/lustre/utils/liblustreapi_chlg.c +++ b/lustre/utils/liblustreapi_chlg.c @@ -247,6 +247,8 @@ int llapi_changelog_recv(void *priv, struct changelog_rec **rech) rec_fmt |= CLF_EXTRA_FLAGS; if (cp->clp_send_extra_flags & CHANGELOG_EXTRA_FLAG_UIDGID) rec_extra_fmt |= CLFE_UIDGID; + if (cp->clp_send_extra_flags & CHANGELOG_EXTRA_FLAG_NID) + rec_extra_fmt |= CLFE_NID; } if (cp->clp_buf + cp->clp_buf_len <= cp->clp_buf_pos) { diff --git a/lustre/utils/lustre_rsync.c b/lustre/utils/lustre_rsync.c index b78afab..20a1c14 100644 --- a/lustre/utils/lustre_rsync.c +++ b/lustre/utils/lustre_rsync.c @@ -1557,7 +1557,8 @@ int lr_replicate() } rc = llapi_changelog_set_xflags(changelog_priv, - CHANGELOG_EXTRA_FLAG_UIDGID); + CHANGELOG_EXTRA_FLAG_UIDGID | + CHANGELOG_EXTRA_FLAG_NID); if (rc < 0) { fprintf(stderr, "Error setting xflag in changelog for fs %s.\n", status->ls_source_fs); -- 1.8.3.1