From: Amir Shehata Date: Tue, 24 Mar 2020 22:18:18 +0000 (-0700) Subject: LU-13385 lnet: Calculate IB service-id X-Git-Tag: 2.13.54~228 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=19420ef72262bd7e4a44b5e682a18ef808f7a922 LU-13385 lnet: Calculate IB service-id The service-id is determined by the: 1. port space (currently hard coded to TCP) 2. service port number (defaults to 987) options ko2iblnd service= Can be used to change the service port number. However, when configuring QoS, we need to know the service-id, which can then be mapped to the service level in the opensm's qos-policy.conf file. This patch adds a new lnetctl command which returns the service-id. lnetctl service-id Test-Parameters: trivial Signed-off-by: Amir Shehata Change-Id: Ifab46473088239a391314040bac91d0f791ea3e4 Reviewed-on: https://review.whamcloud.com/38045 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: Olaf Weber Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index ebd3400..961808b 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -749,6 +749,25 @@ EXTRA_KCFLAGS="$tmp_flags" ]) # LN_IB_SG_DMA_ADDRESS_EXISTS # +# LN_USR_RDMA +# +# +AC_DEFUN([LN_USR_RDMA], [ +AC_MSG_CHECKING([if RDMA_PS_TCP exists]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + #include + + int main(void) { + int x = (int)RDMA_PS_TCP; + return x; + } +])],[ + AC_DEFINE(HAVE_USRSPC_RDMA_PS_TCP, 1, + [RDMA_PS_TCP exists]) +]) +]) # LN_USR_RDMA + +# # LN_PROG_LINUX # # LNet linux kernel checks @@ -815,6 +834,7 @@ AS_IF([test "$enable_efence" = yes], [ AC_SUBST(LIBEFENCE) LN_CONFIG_DLC +LN_USR_RDMA ]) # LN_CONFIGURE # diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 4a64e6e..5714b5c 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -50,6 +50,7 @@ #include #include #include +#include #include "liblnetconfig.h" #define CONFIG_CMD "configure" @@ -63,8 +64,13 @@ #define MAX_NUM_IPS 128 #define modparam_path "/sys/module/lnet/parameters/" +#define o2ib_modparam_path "/sys/module/ko2iblnd/parameters/" #define gni_nid_path "/proc/cray_xt/" +#ifndef HAVE_USRSPC_RDMA_PS_TCP +#define RDMA_PS_TCP 0x0106 +#endif + const char *gmsg_stat_names[] = {"sent_stats", "received_stats", "dropped_stats"}; @@ -3202,7 +3208,7 @@ static void add_to_global(struct cYAML *show_rc, struct cYAML *node, } static int build_global_yaml_entry(char *err_str, int err_len, int seq_no, - char *name, __u32 value, + char *name, __u64 value, struct cYAML **show_rc, struct cYAML **err_rc, int err) { @@ -3398,6 +3404,28 @@ int lustre_lnet_show_retry_count(int seq_no, struct cYAML **show_rc, err_rc, l_errno); } +int lustre_lnet_calc_service_id(__u64 *service_id) +{ + int rc = LUSTRE_CFG_RC_OUT_OF_MEM; + char val[LNET_MAX_STR_LEN]; + int service_port = -1, l_errno = 0; + + rc = read_sysfs_file(o2ib_modparam_path, "service", val, + 1, sizeof(val)); + if (rc) { + l_errno = errno; + fprintf(stderr, "error:\n msg: \"cannot get service port: %s (%d)\"\n", + strerror(l_errno), -l_errno); + return rc; + } else { + service_port = atoi(val); + } + + *service_id = htobe64(((__u64)RDMA_PS_TCP << 16) + service_port); + + return LUSTRE_CFG_RC_NO_ERR; +} + int show_recovery_queue(enum lnet_health_type type, char *name, int seq_no, struct cYAML **show_rc, struct cYAML **err_rc) { diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index 2e5aff3..176e1cd 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -424,6 +424,12 @@ int lustre_lnet_show_max_intf(int seq_no, struct cYAML **show_rc, struct cYAML **err_rc); /* + * lustre_lnet_calc_service_id + * Calculate the lustre service id to be used for qos + */ +int lustre_lnet_calc_service_id(__u64 *service_id); + +/* * lustre_lnet_config_discovery * Enable or disable peer discovery. Peer discovery is enabled by default. * diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index e4c9f19..5c23c5c 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -82,6 +82,7 @@ static int jt_global(int argc, char **argv); static int jt_peers(int argc, char **argv); static int jt_set_ni_value(int argc, char **argv); static int jt_set_peer_ni_value(int argc, char **argv); +static int jt_calc_service_id(int argc, char **argv); command_t cmd_list[] = { {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all]"}, @@ -99,6 +100,7 @@ command_t cmd_list[] = { {"peer", jt_peers, 0, "peer {add | del | show | help}"}, {"ping", jt_ping, 0, "ping nid,[nid,...]"}, {"discover", jt_discover, 0, "discover nid[,nid,...]"}, + {"service-id", jt_calc_service_id, 0, "Calculate IB Lustre service ID\n"}, {"help", Parser_help, 0, "help"}, {"exit", Parser_quit, 0, "quit"}, {"quit", Parser_quit, 0, "quit"}, @@ -242,6 +244,24 @@ command_t peer_cmds[] = { { 0, 0, 0, NULL } }; +static int jt_calc_service_id(int argc, char **argv) +{ + int rc; + __u64 service_id; + + rc = lustre_lnet_calc_service_id(&service_id); + if (rc != LUSTRE_CFG_RC_NO_ERR) + return rc; + + /* + * cYAML currently doesn't support printing hex values. + * Therefore just print it locally here + */ + printf("service id:\n value: 0x%llx\n", service_id); + + return rc; +} + static inline void print_help(const command_t cmds[], const char *cmd_type, const char *pc_name) {