From: Emoly Liu Date: Thu, 9 Oct 2014 16:50:00 +0000 (+0800) Subject: LU-3397 lprocfs: create "export" /proc file on server X-Git-Tag: 2.10.58~25 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=b4b773466a0f3a178364e62b8549de08a3f9ce32;p=fs%2Flustre-release.git LU-3397 lprocfs: create "export" /proc file on server Similar to the "import" file on the client for each client-to-server connection, it would be useful to have a file on the server in the per-nid directory obdfilter/*/exports/$NID/export. This contains export connection information as in the "import" file, like: a793e354-49c0-aa11-8c4f-a4f2b1a1a92b: name: MGS client: 10.211.55.10@tcp connect_flags: [ version, barrier, adaptive_timeouts, ... ] connect_data: flags: 0x2000011005002020 instance: 0 target_version: 2.10.51.0 export_flags: [ ... ] Also, sanity.sh test_0d is added to verify this patch. Signed-off-by: Emoly Liu Signed-off-by: James Simmons Signed-off-by: Andreas Dilger Change-Id: I60896090e3a8ad872141a8d4299f0698f0a5636a Reviewed-on: https://review.whamcloud.com/6713 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Nathaniel Clark Reviewed-by: James Simmons --- diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index 4b244f3..f253f68 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -59,6 +59,22 @@ struct lprocfs_vars { mode_t proc_mode; }; +/** + * Append a space separated list of current set flags to str. + */ +#define flag2str(port, flag) \ + do { \ + if ((port)->port##_##flag) { \ + seq_printf(m, "%s" #flag, first ? "" : ", "); \ + first = false; \ + } \ + } while (0) + +void obd_connect_seq_flags2str(struct seq_file *m, __u64 flags, __u64 flags2, + const char *sep); +void obd_connect_data_seqprint(struct seq_file *m, + struct obd_connect_data *ocd); + /* if we find more consumers this could be generalized */ #define OBD_HIST_MAX 32 struct obd_histogram { diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 57426d9..2bac82f 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -5599,12 +5599,12 @@ static int mdt_connect_internal(const struct lu_env *env, data->ocd_grant_max_blks = ddp->ddp_max_extent_blks; } - if (OCD_HAS_FLAG(data, GRANT)) { - /* Save connect_data we have so far because tgt_grant_connect() - * uses it to calculate grant. */ - exp->exp_connect_data = *data; + /* Save connect_data we have so far because tgt_grant_connect() + * uses it to calculate grant, and we want to save the client + * version before it is overwritten by LUSTRE_VERSION_CODE. */ + exp->exp_connect_data = *data; + if (OCD_HAS_FLAG(data, GRANT)) tgt_grant_connect(env, exp, data, !reconnect); - } if (OCD_HAS_FLAG(data, MAXBYTES)) data->ocd_maxbytes = mdt->mdt_lut.lut_dt_conf.ddp_maxbytes; diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 75b9c1b..20ccd7e 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -1615,28 +1615,16 @@ static struct lu_device_type mgs_device_type = { .ldt_ctx_tags = LCT_MG_THREAD }; -static int mgs_obd_connect(const struct lu_env *env, struct obd_export **exp, - struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *data, void *localdata) +static int mgs_obd_reconnect(const struct lu_env *env, struct obd_export *exp, + struct obd_device *obd, struct obd_uuid *cluuid, + struct obd_connect_data *data, void *localdata) { - struct obd_export *lexp; - struct lustre_handle conn = { - .cookie = 0, - }; - int rc; - ENTRY; if (exp == NULL || obd == NULL || cluuid == NULL) RETURN(-EINVAL); - rc = class_connect(&conn, obd, cluuid); - if (rc) - RETURN(rc); - - lexp = class_conn2export(&conn); - if (lexp == NULL) - RETURN(-EFAULT); + tgt_counter_incr(exp, LPROC_MGS_CONNECT); if (data != NULL) { data->ocd_connect_flags &= MGS_CONNECT_SUPPORTED; @@ -1644,43 +1632,48 @@ static int mgs_obd_connect(const struct lu_env *env, struct obd_export **exp, if (data->ocd_connect_flags & OBD_CONNECT_FLAGS2) data->ocd_connect_flags2 &= MGS_CONNECT_SUPPORTED2; + exp->exp_connect_data = *data; data->ocd_version = LUSTRE_VERSION_CODE; - lexp->exp_connect_data = *data; } - tgt_counter_incr(lexp, LPROC_MGS_CONNECT); - - rc = mgs_export_stats_init(obd, lexp, localdata); - if (rc) - class_disconnect(lexp); - else - *exp = lexp; - - RETURN(rc); + RETURN(mgs_export_stats_init(obd, exp, localdata)); } -static int mgs_obd_reconnect(const struct lu_env *env, struct obd_export *exp, - struct obd_device *obd, struct obd_uuid *cluuid, - struct obd_connect_data *data, void *localdata) +static int mgs_obd_connect(const struct lu_env *env, struct obd_export **exp, + struct obd_device *obd, struct obd_uuid *cluuid, + struct obd_connect_data *data, void *localdata) { + struct obd_export *lexp; + struct lustre_handle conn = { + .cookie = 0, + }; + int rc; + ENTRY; if (exp == NULL || obd == NULL || cluuid == NULL) RETURN(-EINVAL); - tgt_counter_incr(exp, LPROC_MGS_CONNECT); + rc = class_connect(&conn, obd, cluuid); + if (rc) + RETURN(rc); - if (data != NULL) { - data->ocd_connect_flags &= MGS_CONNECT_SUPPORTED; + lexp = class_conn2export(&conn); + if (lexp == NULL) + RETURN(-EFAULT); - if (data->ocd_connect_flags & OBD_CONNECT_FLAGS2) - data->ocd_connect_flags2 &= MGS_CONNECT_SUPPORTED2; + rc = mgs_obd_reconnect(env, lexp, obd, cluuid, data, localdata); + if (rc) + GOTO(out_disconnect, rc); - data->ocd_version = LUSTRE_VERSION_CODE; - exp->exp_connect_data = *data; - } + *exp = lexp; - RETURN(mgs_export_stats_init(obd, exp, localdata)); + RETURN(rc); + +out_disconnect: + class_disconnect(lexp); + + return rc; } static int mgs_obd_disconnect(struct obd_export *exp) diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index 86a0a01..967f788 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -758,16 +758,6 @@ void lprocfs_stats_collect(struct lprocfs_stats *stats, int idx, lprocfs_stats_unlock(stats, LPROCFS_GET_NUM_CPU, &flags); } -/** - * Append a space separated list of current set flags to str. - */ -#define flag2str(flag) \ - do { \ - if (imp->imp_##flag) { \ - seq_printf(m, "%s" #flag, first ? "" : ", "); \ - first = false; \ - } \ - } while (0) static void obd_import_flags2str(struct obd_import *imp, struct seq_file *m) { bool first = true; @@ -777,18 +767,17 @@ static void obd_import_flags2str(struct obd_import *imp, struct seq_file *m) first = false; } - flag2str(invalid); - flag2str(deactive); - flag2str(replayable); - flag2str(delayed_recovery); - flag2str(vbr_failed); - flag2str(pingable); - flag2str(resend_replay); - flag2str(no_pinger_recover); - flag2str(need_mne_swab); - flag2str(connect_tried); + flag2str(imp, invalid); + flag2str(imp, deactive); + flag2str(imp, replayable); + flag2str(imp, delayed_recovery); + flag2str(imp, vbr_failed); + flag2str(imp, pingable); + flag2str(imp, resend_replay); + flag2str(imp, no_pinger_recover); + flag2str(imp, need_mne_swab); + flag2str(imp, connect_tried); } -#undef flag2str static const char *obd_connect_names[] = { /* flags names */ @@ -862,8 +851,8 @@ static const char *obd_connect_names[] = { NULL }; -static void obd_connect_seq_flags2str(struct seq_file *m, __u64 flags, - __u64 flags2, const char *sep) +void obd_connect_seq_flags2str(struct seq_file *m, __u64 flags, __u64 flags2, + const char *sep) { bool first = true; __u64 mask; @@ -900,6 +889,7 @@ static void obd_connect_seq_flags2str(struct seq_file *m, __u64 flags, first = false; } } +EXPORT_SYMBOL(obd_connect_seq_flags2str); int obd_connect_flags2str(char *page, int count, __u64 flags, __u64 flags2, const char *sep) @@ -936,8 +926,8 @@ int obd_connect_flags2str(char *page, int count, __u64 flags, __u64 flags2, } EXPORT_SYMBOL(obd_connect_flags2str); -static void obd_connect_data_seqprint(struct seq_file *m, - struct obd_connect_data *ocd) +void +obd_connect_data_seqprint(struct seq_file *m, struct obd_connect_data *ocd) { __u64 flags; diff --git a/lustre/obdclass/lprocfs_status_server.c b/lustre/obdclass/lprocfs_status_server.c index 242d3fa..08f3f18 100644 --- a/lustre/obdclass/lprocfs_status_server.c +++ b/lustre/obdclass/lprocfs_status_server.c @@ -115,6 +115,86 @@ int lprocfs_num_exports_seq_show(struct seq_file *m, void *data) } EXPORT_SYMBOL(lprocfs_num_exports_seq_show); +static int obd_export_flags2str(struct obd_export *exp, struct seq_file *m) +{ + bool first = true; + + flag2str(exp, failed); + flag2str(exp, in_recovery); + flag2str(exp, disconnected); + flag2str(exp, connecting); + + return 0; +} + +static int +lprocfs_exp_print_export_seq(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, void *cb_data) +{ + struct seq_file *m = cb_data; + struct obd_export *exp = cfs_hash_object(hs, hnode); + struct obd_device *obd; + struct obd_connect_data *ocd; + + LASSERT(exp != NULL); + if (exp->exp_nid_stats == NULL) + goto out; + obd = exp->exp_obd; + ocd = &exp->exp_connect_data; + + seq_printf(m, "%s:\n" + " name: %s\n" + " client: %s\n" + " connect_flags: [ ", + obd_uuid2str(&exp->exp_client_uuid), + obd->obd_name, + obd_export_nid2str(exp)); + obd_connect_seq_flags2str(m, ocd->ocd_connect_flags, + ocd->ocd_connect_flags2, ", "); + seq_printf(m, " ]\n"); + obd_connect_data_seqprint(m, ocd); + seq_printf(m, " export_flags: [ "); + obd_export_flags2str(exp, m); + seq_printf(m, " ]\n"); + +out: + return 0; +} + +/** + * RPC connections are composed of an import and an export. Using the + * lctl utility we can extract important information about the state. + * The lprocfs_exp_export_seq_show routine displays the state information + * for the export. + * + * \param[in] m seq file + * \param[in] data unused + * + * \retval 0 on success + * + * The format of the export state information is like: + * a793e354-49c0-aa11-8c4f-a4f2b1a1a92b: + * name: MGS + * client: 10.211.55.10@tcp + * connect_flags: [ version, barrier, adaptive_timeouts, ... ] + * connect_data: + * flags: 0x2000011005002020 + * instance: 0 + * target_version: 2.10.51.0 + * export_flags: [ ... ] + * + */ +static int lprocfs_exp_export_seq_show(struct seq_file *m, void *data) +{ + struct nid_stat *stats = m->private; + struct obd_device *obd = stats->nid_obd; + + cfs_hash_for_each_key(obd->obd_nid_hash, &stats->nid, + lprocfs_exp_print_export_seq, m); + return 0; +} +LPROC_SEQ_FOPS_RO(lprocfs_exp_export); + static void lprocfs_free_client_stats(struct nid_stat *client_stat) { CDEBUG(D_CONFIG, "stat %p - data %p/%p\n", client_stat, @@ -385,7 +465,8 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid) &lprocfs_exp_nodemap_fops); if (IS_ERR(entry)) { rc = PTR_ERR(entry); - CWARN("Error adding the nodemap file: rc = %d\n", rc); + CWARN("%s: error adding the nodemap file: rc = %d\n", + obd->obd_name, rc); GOTO(destroy_new_ns, rc); } @@ -393,7 +474,8 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid) &lprocfs_exp_uuid_fops); if (IS_ERR(entry)) { rc = PTR_ERR(entry); - CWARN("Error adding the NID stats file: rc = %d\n", rc); + CWARN("%s: error adding the NID stats file: rc = %d\n", + obd->obd_name, rc); GOTO(destroy_new_ns, rc); } @@ -401,7 +483,17 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid) &lprocfs_exp_hash_fops); if (IS_ERR(entry)) { rc = PTR_ERR(entry); - CWARN("Error adding the hash file: rc = %d\n", rc); + CWARN("%s: error adding the hash file: rc = %d\n", + obd->obd_name, rc); + GOTO(destroy_new_ns, rc); + } + + entry = lprocfs_add_simple(new_stat->nid_proc, "export", + new_stat, &lprocfs_exp_export_fops); + if (IS_ERR(entry)) { + rc = PTR_ERR(entry); + CWARN("%s: error adding the export file: rc = %d\n", + obd->obd_name, rc); GOTO(destroy_new_ns, rc); } @@ -409,7 +501,7 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid) &lprocfs_exp_replydata_fops); if (IS_ERR(entry)) { rc = PTR_ERR(entry); - CWARN("%s: Error adding the reply_data file: rc = %d\n", + CWARN("%s: error adding the reply_data file: rc = %d\n", obd->obd_name, rc); GOTO(destroy_new_ns, rc); } diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index dba54a4..9693692 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -171,8 +171,6 @@ static int ofd_parse_connect_data(const struct lu_env *env, if (data->ocd_connect_flags & OBD_CONNECT_FLAGS2) data->ocd_connect_flags2 &= OST_CONNECT_SUPPORTED2; - data->ocd_version = LUSTRE_VERSION_CODE; - /* Kindly make sure the SKIP_ORPHAN flag is from MDS. */ if (data->ocd_connect_flags & OBD_CONNECT_MDS) CDEBUG(D_HA, "%s: Received MDS connection for group %u\n", @@ -214,12 +212,12 @@ static int ofd_parse_connect_data(const struct lu_env *env, data->ocd_grant_max_blks = ddp->ddp_max_extent_blks; } - if (OCD_HAS_FLAG(data, GRANT)) { - /* Save connect_data we have so far because tgt_grant_connect() - * uses it to calculate grant. */ - exp->exp_connect_data = *data; + /* Save connect_data we have so far because tgt_grant_connect() + * uses it to calculate grant, and we want to save the client + * version before it is overwritten by LUSTRE_VERSION_CODE. */ + exp->exp_connect_data = *data; + if (OCD_HAS_FLAG(data, GRANT)) tgt_grant_connect(env, exp, data, new_connection); - } if (data->ocd_connect_flags & OBD_CONNECT_INDEX) { struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd; @@ -272,6 +270,8 @@ static int ofd_parse_connect_data(const struct lu_env *env, if (data->ocd_connect_flags & OBD_CONNECT_MAXBYTES) data->ocd_maxbytes = ofd->ofd_lut.lut_dt_conf.ddp_maxbytes; + data->ocd_version = LUSTRE_VERSION_CODE; + if (OCD_HAS_FLAG(data, PINGLESS)) { if (ptlrpc_pinger_suppress_pings()) { spin_lock(&exp->exp_obd->obd_dev_lock); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index aec099a..4b61b27 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -170,6 +170,45 @@ test_0c() { } run_test 0c "check import proc" +test_0d() { # LU-3397 + [ $(lustre_version_code mgs) -lt $(version_code 2.10.57) ] && + skip "proc exports not supported before 2.10.57" && return + + local mgs_exp="mgs.MGS.exports" + local client_uuid=$($LCTL get_param -n mgc.*.uuid) + local exp_client_nid + local exp_client_version + local exp_val + local imp_val + local temp_imp=$DIR/$tfile.import + local temp_exp=$DIR/$tfile.export + + # save mgc import file to $temp_imp + $LCTL get_param mgc.*.import | tee $temp_imp + # Check if client uuid is found in MGS export + for exp_client_nid in $(do_facet mgs $LCTL get_param -N $mgs_exp.*); do + [ $(do_facet mgs $LCTL get_param $exp_client_nid.uuid) == \ + $client_uuid ] && + break; + done + # save mgs export file to $temp_exp + do_facet mgs $LCTL get_param $exp_client_nid.export | tee $temp_exp + + # Compare the value of field "connect_flags" + imp_val=$(grep "connect_flags" $temp_imp) + exp_val=$(grep "connect_flags" $temp_exp) + [ "$exp_val" == "$imp_val" ] || + error "export flags '$exp_val' != import flags '$imp_val'" + + # Compare the value of client version + exp_client_version=$(awk '/target_version:/ { print $2 }' $temp_exp) + exp_val=$(version_code $exp_client_version) + imp_val=$(lustre_version_code client) + [ "$exp_val" == "$imp_val" ] || + error "export client version '$exp_val' != '$imp_val'" +} +run_test 0d "check export proc =============================" + test_1() { test_mkdir $DIR/$tdir test_mkdir $DIR/$tdir/d2