From 5c999e931712f516d1c04c1e061da72dee2dc3ba Mon Sep 17 00:00:00 2001 From: Christopher Morrone Date: Thu, 20 Jan 2011 13:24:48 +0100 Subject: [PATCH] b=21456 Patch to support lnet v1 pings in 'lctl ping' i=maxim i=isaac --- lnet/ChangeLog | 4 ++ lnet/include/lnet/lib-lnet.h | 2 +- lnet/include/lnet/lib-types.h | 50 +++++++++++++++++------ lnet/lnet/api-ni.c | 68 ++++++++------------------------ lnet/lnet/router.c | 92 +++++++++++++++++++++++++++++++------------ 5 files changed, 126 insertions(+), 90 deletions(-) diff --git a/lnet/ChangeLog b/lnet/ChangeLog index 69c7f8c..bd52cb2 100644 --- a/lnet/ChangeLog +++ b/lnet/ChangeLog @@ -16,6 +16,10 @@ Severity : normal Bugzilla : 23575 Description: fix o2iblnd v2 regression of credit deadlock with v1 peers (bug 14425). +Severity : normal +Bugzilla : 21456 +Description: lctl ping of 1.6.6 server from 1.8 router causes protocol error + ------------------------------------------------------------------------------- 2010-10-29 Oracle, Inc. diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 11c4e3d..9ef1207 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -781,8 +781,8 @@ int lnet_peer_buffer_credits(lnet_ni_t *ni); int lnet_router_checker_start(void); void lnet_router_checker_stop(void); -void lnet_swap_pinginfo(lnet_ping_info_t *info); int lnet_router_down_ni(lnet_peer_t *rtr, __u32 net); +int lnet_parse_pinginfo(lnet_ping_info_t *info, int len, int expected_nids); int lnet_ping_target_init(void); void lnet_ping_target_fini(void); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 92b1295..d7835a0 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -383,15 +383,49 @@ typedef struct lnet_lnd #endif } lnd_t; +#define LNET_PROTO_PING_MATCHBITS 0x8000000000000000LL +#define LNET_PROTO_PING_VERSION 2 +#define LNET_PROTO_PING_VERSION1 1 + #define LNET_NI_STATUS_UP 0x15aac0de #define LNET_NI_STATUS_DOWN 0xdeadface #define LNET_NI_STATUS_INVALID 0x00000000 + typedef struct { lnet_nid_t ns_nid; __u32 ns_status; __u32 ns_unused; } WIRE_ATTR lnet_ni_status_t; +typedef struct { + __u32 pi_magic; + __u32 pi_version; + lnet_pid_t pi_pid; + __u32 pi_nnis; +#define pi_ni pi_body.pb_ni +#define pi_nid pi_body.pb_nid + + union { + lnet_nid_t pb_nid[0]; /* LNET_PROTO_PING_VERSION1 */ + lnet_ni_status_t pb_ni[0]; /* LNET_PROTO_PING_VERSION */ + } pi_body; +} WIRE_ATTR lnet_ping_info_t; + +static inline size_t +lnet_pinginfo_size_v(int n_ids, int version) +{ + LASSERT (n_ids >= 0); + LASSERT (version == LNET_PROTO_PING_VERSION || + version == LNET_PROTO_PING_VERSION1); + + if (version == LNET_PROTO_PING_VERSION) + return offsetof(lnet_ping_info_t, pi_ni[n_ids]); + + return offsetof(lnet_ping_info_t, pi_nid[n_ids]); +} + +#define lnet_pinginfo_size(n) lnet_pinginfo_size_v((n), LNET_PROTO_PING_VERSION) + #define LNET_MAX_INTERFACES 16 typedef struct lnet_ni { @@ -412,20 +446,10 @@ typedef struct lnet_ni { char *ni_interfaces[LNET_MAX_INTERFACES]; /* equivalent interfaces to use */ } lnet_ni_t; -#define LNET_PROTO_PING_MATCHBITS 0x8000000000000000LL -#define LNET_PROTO_PING_VERSION 2 -#define LNET_PROTO_PING_VERSION1 1 -typedef struct { - __u32 pi_magic; - __u32 pi_version; - lnet_pid_t pi_pid; - __u32 pi_nnis; - lnet_ni_status_t pi_ni[0]; -} WIRE_ATTR lnet_ping_info_t; - /* router checker data, per router */ -#define LNET_MAX_RTR_NIS 16 -#define LNET_PINGINFO_SIZE offsetof(lnet_ping_info_t, pi_ni[LNET_MAX_RTR_NIS]) +#define LNET_MAX_RTR_NIS 16 +#define LNET_MAX_PINGINFO_SIZE lnet_pinginfo_size(LNET_MAX_RTR_NIS) + typedef struct { struct list_head rcd_list; /* chain on the_lnet.ln_zombie_rcd */ lnet_handle_md_t rcd_mdh; /* ping buffer MD */ diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 657568f..ed5f45e 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -1524,7 +1524,7 @@ lnet_create_ping_info(void) LASSERT (rc == 0); } - infosz = offsetof(lnet_ping_info_t, pi_ni[n]); + infosz = lnet_pinginfo_size(n); LIBCFS_ALLOC(pinfo, infosz); if (pinfo == NULL) { CERROR("Can't allocate ping info[%d]\n", n); @@ -1574,8 +1574,7 @@ lnet_destroy_ping_info(void) LNET_UNLOCK(); LIBCFS_FREE(the_lnet.ln_ping_info, - offsetof(lnet_ping_info_t, - pi_ni[the_lnet.ln_ping_info->pi_nnis])); + lnet_pinginfo_size(the_lnet.ln_ping_info->pi_nnis)); the_lnet.ln_ping_info = NULL; return; } @@ -1586,7 +1585,6 @@ lnet_ping_target_init(void) lnet_handle_me_t meh; int rc; int rc2; - int infosz; rc = lnet_create_ping_info(); if (rc != 0) @@ -1611,11 +1609,9 @@ lnet_ping_target_init(void) goto failed_1; } - infosz = offsetof(lnet_ping_info_t, - pi_ni[the_lnet.ln_ping_info->pi_nnis]); rc = LNetMDAttach(meh, (lnet_md_t){.start = the_lnet.ln_ping_info, - .length = infosz, + .length = lnet_pinginfo_size(the_lnet.ln_ping_info->pi_nnis), .threshold = LNET_MD_THRESH_INF, .options = (LNET_MD_OP_GET | LNET_MD_TRUNCATE | @@ -1688,11 +1684,10 @@ lnet_ping (lnet_process_id_t id, int timeout_ms, lnet_process_id_t *ids, int n_i int unlinked = 0; int replied = 0; const int a_long_time = 60000; /* mS */ - int infosz = offsetof(lnet_ping_info_t, pi_ni[n_ids]); + int infosz; lnet_ping_info_t *info; lnet_process_id_t tmpid; int i; - int nob; int rc; int rc2; cfs_sigset_t blocked; @@ -1706,6 +1701,7 @@ lnet_ping (lnet_process_id_t id, int timeout_ms, lnet_process_id_t *ids, int n_i if (id.pid == LNET_PID_ANY) id.pid = LUSTRE_SRV_LNET_PID; + infosz = lnet_pinginfo_size(n_ids); LIBCFS_ALLOC(info, infosz); if (info == NULL) return -ENOMEM; @@ -1788,58 +1784,28 @@ lnet_ping (lnet_process_id_t id, int timeout_ms, lnet_process_id_t *ids, int n_i if (!replied) { if (rc >= 0) - CWARN("%s: Unexpected rc >= 0 but no reply!\n", - libcfs_id2str(id)); + CWARN("%s: Unexpected rc(%d) >= 0 but no reply!\n", + libcfs_id2str(id), rc); rc = -EIO; goto out_1; } - nob = rc; - LASSERT (nob >= 0 && nob <= infosz); - - rc = -EPROTO; /* if I can't parse... */ - - if (nob < 8) { - /* can't check magic/version */ - CERROR("%s: ping info too short %d\n", - libcfs_id2str(id), nob); - goto out_1; - } - - if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) { - lnet_swap_pinginfo(info); - } else if (info->pi_magic != LNET_PROTO_PING_MAGIC) { - CERROR("%s: Unexpected magic %08x\n", - libcfs_id2str(id), info->pi_magic); - goto out_1; - } - - if (info->pi_version != LNET_PROTO_PING_VERSION) { - CERROR("%s: Unexpected version 0x%x\n", - libcfs_id2str(id), info->pi_version); - goto out_1; - } - - if (nob < offsetof(lnet_ping_info_t, pi_ni[0])) { - CERROR("%s: Short reply %d(%d min)\n", libcfs_id2str(id), - nob, (int)offsetof(lnet_ping_info_t, pi_ni[0])); - goto out_1; - } - - if (info->pi_nnis < n_ids) - n_ids = info->pi_nnis; - - if (nob < offsetof(lnet_ping_info_t, pi_ni[n_ids])) { - CERROR("%s: Short reply %d(%d expected)\n", libcfs_id2str(id), - nob, (int)offsetof(lnet_ping_info_t, pi_ni[n_ids])); + LASSERT (rc >= 0 && rc <= infosz); /* rc == event.mlength */ + rc = lnet_parse_pinginfo(info, rc, n_ids); + if (rc != 0) { + CERROR("Bad ping reply from %s\n", libcfs_id2str(id)); goto out_1; } rc = -EFAULT; /* If I SEGV... */ - + n_ids = MIN(info->pi_nnis, n_ids); for (i = 0; i < n_ids; i++) { tmpid.pid = info->pi_pid; - tmpid.nid = info->pi_ni[i].ns_nid; + if (info->pi_version == LNET_PROTO_PING_VERSION) + tmpid.nid = info->pi_ni[i].ns_nid; + else + tmpid.nid = info->pi_nid[i]; + #ifdef __KERNEL__ if (copy_to_user(&ids[i], &tmpid, sizeof(tmpid))) goto out_1; diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index 05c2027..8c7b569 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -502,21 +502,69 @@ lnet_get_route (int idx, __u32 *net, __u32 *hops, return -ENOENT; } -void -lnet_swap_pinginfo(lnet_ping_info_t *info) +int +lnet_parse_pinginfo(lnet_ping_info_t *info, int len, int expected_nids) { - int i; - lnet_ni_status_t *stat; + int rc = -EPROTO; /* if I can't parse... */ + int i, swab, n_nids; + + if (len < 8) { /* can't check magic/version */ + CNETERR("Ping info too short %d\n", len); + return rc; + } + + if (info->pi_magic == LNET_PROTO_PING_MAGIC) { + swab = 0; + } else if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) { + swab = 1; + } else { + CNETERR("Unexpected magic %08x\n", info->pi_magic); + return rc; + } + + if (swab) + __swab32s(&info->pi_version); + + if (info->pi_version != LNET_PROTO_PING_VERSION && + info->pi_version != LNET_PROTO_PING_VERSION1) { + CNETERR("Unexpected version 0x%x\n", info->pi_version); + return rc; + } + + if (len < lnet_pinginfo_size(0)) { /* can't check pid/nnis */ + CNETERR("Short reply %d(%d min)\n", len, + (int)lnet_pinginfo_size(0)); + return rc; + } + + if (swab) { + __swab32s(&info->pi_pid); + __swab32s(&info->pi_nnis); + } + + n_nids = MIN(info->pi_nnis, expected_nids); + if (len < lnet_pinginfo_size_v(n_nids, info->pi_version)) { + CNETERR("Short reply %d(%d expected)\n", len, + (int)lnet_pinginfo_size_v(n_nids, info->pi_version)); + return rc; + } + + if (!swab) + return 0; + + for (i = 0; i < n_nids; i++) { + lnet_ni_status_t *stat; + + if (info->pi_version == LNET_PROTO_PING_VERSION1) { + __swab64s(&info->pi_nid[i]); + continue; + } - __swab32s(&info->pi_version); - __swab32s(&info->pi_pid); - __swab32s(&info->pi_nnis); - for (i = 0; i < info->pi_nnis && i < LNET_MAX_RTR_NIS; i++) { stat = &info->pi_ni[i]; __swab64s(&stat->ns_nid); __swab32s(&stat->ns_status); } - return; + return 0; } /* Returns # of down NIs, or negative error codes; ignore downed NIs @@ -529,6 +577,7 @@ lnet_router_down_ni(lnet_peer_t *rtr, __u32 net) int ptl_up = 0; int ptl_down = 0; lnet_ping_info_t *info; + int rc; if (!avoid_asym_router_failure) return -ENOENT; @@ -542,24 +591,17 @@ lnet_router_down_ni(lnet_peer_t *rtr, __u32 net) info = rtr->lp_rcd->rcd_pinginfo; LASSERT (info != NULL); - /* NB always racing with network! */ - if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) { - lnet_swap_pinginfo(info); - } else if (info->pi_magic != LNET_PROTO_PING_MAGIC) { - CNETERR("%s: Unexpected magic %08x\n", - libcfs_nid2str(rtr->lp_nid), info->pi_magic); + rc = lnet_parse_pinginfo(info, /* NB always racing with network! */ + LNET_MAX_PINGINFO_SIZE, LNET_MAX_RTR_NIS); + if (rc != 0) { + CNETERR("Bad ping reply from %s\n", + libcfs_nid2str(rtr->lp_nid)); return -EPROTO; } if (info->pi_version == LNET_PROTO_PING_VERSION1) return -ENOENT; /* v1 doesn't carry NI status info */ - if (info->pi_version != LNET_PROTO_PING_VERSION) { - CNETERR("%s: Unexpected version 0x%x\n", - libcfs_nid2str(rtr->lp_nid), info->pi_version); - return -EPROTO; - } - for (i = 0; i < info->pi_nnis && i < LNET_MAX_RTR_NIS; i++) { lnet_ni_status_t *stat = &info->pi_ni[i]; lnet_nid_t nid = stat->ns_nid; @@ -745,7 +787,7 @@ lnet_destroy_rc_data (lnet_rc_data_t *rcd) /* detached from network */ LASSERT (LNetHandleIsEqual(rcd->rcd_mdh, LNET_INVALID_HANDLE)); - LIBCFS_FREE(rcd->rcd_pinginfo, LNET_PINGINFO_SIZE); + LIBCFS_FREE(rcd->rcd_pinginfo, LNET_MAX_PINGINFO_SIZE); LIBCFS_FREE(rcd, sizeof(*rcd)); return; } @@ -762,13 +804,13 @@ lnet_create_rc_data (void) if (rcd == NULL) return NULL; - LIBCFS_ALLOC(pi, LNET_PINGINFO_SIZE); + LIBCFS_ALLOC(pi, LNET_MAX_PINGINFO_SIZE); if (pi == NULL) { LIBCFS_FREE(rcd, sizeof(*rcd)); return NULL; } - memset(pi, 0, LNET_PINGINFO_SIZE); + memset(pi, 0, LNET_MAX_PINGINFO_SIZE); for (i = 0; i < LNET_MAX_RTR_NIS; i++) { pi->pi_ni[i].ns_nid = LNET_NID_ANY; pi->pi_ni[i].ns_status = LNET_NI_STATUS_INVALID; @@ -780,7 +822,7 @@ lnet_create_rc_data (void) LASSERT (!LNetHandleIsEqual(the_lnet.ln_rc_eqh, LNET_EQ_NONE)); rc = LNetMDBind((lnet_md_t){.start = pi, .user_ptr = rcd, - .length = LNET_PINGINFO_SIZE, + .length = LNET_MAX_PINGINFO_SIZE, .threshold = LNET_MD_THRESH_INF, .options = LNET_MD_TRUNCATE, .eq_handle = the_lnet.ln_rc_eqh}, -- 1.8.3.1