LU-13385 lnet: Calculate IB service-id

The service-id is determined by the:
1. port space (currently hard coded to TCP)
2. service port number (defaults to 987)

options ko2iblnd service=<num>

Can be used to change the service port number. However, when
configuring QoS, we need to know the service-id, which can then
be mapped to the service level in the opensm's qos-policy.conf file.

This patch adds a new lnetctl command which returns the service-id.

lnetctl service-id

Test-Parameters: trivial
Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Change-Id: Ifab46473088239a391314040bac91d0f791ea3e4
Reviewed-on: https://review.whamcloud.com/38045
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index ebd3400..961808b 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -749,6 +749,25 @@ EXTRA_KCFLAGS="$tmp_flags"
 ]) # LN_IB_SG_DMA_ADDRESS_EXISTS
 
 #
+# LN_USR_RDMA
+#
+#
+AC_DEFUN([LN_USR_RDMA], [
+AC_MSG_CHECKING([if RDMA_PS_TCP exists])
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([
+       #include <rdma/rdma_user_cm.h>
+
+       int main(void) {
+               int x = (int)RDMA_PS_TCP;
+               return x;
+       }
+])],[
+       AC_DEFINE(HAVE_USRSPC_RDMA_PS_TCP, 1,
+               [RDMA_PS_TCP exists])
+])
+]) # LN_USR_RDMA
+
+#
 # LN_PROG_LINUX
 #
 # LNet linux kernel checks
@@ -815,6 +834,7 @@ AS_IF([test "$enable_efence" = yes], [
 AC_SUBST(LIBEFENCE)
 
 LN_CONFIG_DLC
+LN_USR_RDMA
 ]) # LN_CONFIGURE
 
 #

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index 4a64e6e..5714b5c 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -50,6 +50,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <ifaddrs.h>
+#include <rdma/rdma_user_cm.h>
 #include "liblnetconfig.h"
 
 #define CONFIG_CMD             "configure"
@@ -63,8 +64,13 @@
 #define MAX_NUM_IPS            128
 
 #define modparam_path "/sys/module/lnet/parameters/"
+#define o2ib_modparam_path "/sys/module/ko2iblnd/parameters/"
 #define gni_nid_path "/proc/cray_xt/"
 
+#ifndef HAVE_USRSPC_RDMA_PS_TCP
+#define RDMA_PS_TCP 0x0106
+#endif
+
 const char *gmsg_stat_names[] = {"sent_stats", "received_stats",
                                 "dropped_stats"};
 
@@ -3202,7 +3208,7 @@ static void add_to_global(struct cYAML *show_rc, struct cYAML *node,
 }
 
 static int build_global_yaml_entry(char *err_str, int err_len, int seq_no,
-                                  char *name, __u32 value,
+                                  char *name, __u64 value,
                                   struct cYAML **show_rc,
                                   struct cYAML **err_rc, int err)
 {
@@ -3398,6 +3404,28 @@ int lustre_lnet_show_retry_count(int seq_no, struct cYAML **show_rc,
                                       err_rc, l_errno);
 }
 
+int lustre_lnet_calc_service_id(__u64 *service_id)
+{
+       int rc = LUSTRE_CFG_RC_OUT_OF_MEM;
+       char val[LNET_MAX_STR_LEN];
+       int service_port = -1, l_errno = 0;
+
+       rc = read_sysfs_file(o2ib_modparam_path, "service", val,
+                            1, sizeof(val));
+       if (rc) {
+               l_errno = errno;
+               fprintf(stderr, "error:\n    msg: \"cannot get service port: %s (%d)\"\n",
+                       strerror(l_errno), -l_errno);
+               return rc;
+       } else {
+               service_port = atoi(val);
+       }
+
+       *service_id = htobe64(((__u64)RDMA_PS_TCP << 16) + service_port);
+
+       return LUSTRE_CFG_RC_NO_ERR;
+}
+
 int show_recovery_queue(enum lnet_health_type type, char *name, int seq_no,
                        struct cYAML **show_rc, struct cYAML **err_rc)
 {

diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h
index 2e5aff3..176e1cd 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.h
+++ b/lnet/utils/lnetconfig/liblnetconfig.h
@@ -424,6 +424,12 @@ int lustre_lnet_show_max_intf(int seq_no, struct cYAML **show_rc,
                              struct cYAML **err_rc);
 
 /*
+ * lustre_lnet_calc_service_id
+ *    Calculate the lustre service id to be used for qos
+ */
+int lustre_lnet_calc_service_id(__u64 *service_id);
+
+/*
  * lustre_lnet_config_discovery
  *   Enable or disable peer discovery. Peer discovery is enabled by default.
  *

diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c
index e4c9f19..5c23c5c 100644 (file)
--- a/lnet/utils/lnetctl.c
+++ b/lnet/utils/lnetctl.c
@@ -82,6 +82,7 @@ static int jt_global(int argc, char **argv);
 static int jt_peers(int argc, char **argv);
 static int jt_set_ni_value(int argc, char **argv);
 static int jt_set_peer_ni_value(int argc, char **argv);
+static int jt_calc_service_id(int argc, char **argv);
 
 command_t cmd_list[] = {
        {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all]"},
@@ -99,6 +100,7 @@ command_t cmd_list[] = {
        {"peer", jt_peers, 0, "peer {add | del | show | help}"},
        {"ping", jt_ping, 0, "ping nid,[nid,...]"},
        {"discover", jt_discover, 0, "discover nid[,nid,...]"},
+       {"service-id", jt_calc_service_id, 0, "Calculate IB Lustre service ID\n"},
        {"help", Parser_help, 0, "help"},
        {"exit", Parser_quit, 0, "quit"},
        {"quit", Parser_quit, 0, "quit"},
@@ -242,6 +244,24 @@ command_t peer_cmds[] = {
        { 0, 0, 0, NULL }
 };
 
+static int jt_calc_service_id(int argc, char **argv)
+{
+       int rc;
+       __u64 service_id;
+
+       rc = lustre_lnet_calc_service_id(&service_id);
+       if (rc != LUSTRE_CFG_RC_NO_ERR)
+               return rc;
+
+       /*
+        * cYAML currently doesn't support printing hex values.
+        * Therefore just print it locally here
+        */
+       printf("service id:\n    value: 0x%llx\n", service_id);
+
+       return rc;
+}
+
 static inline void print_help(const command_t cmds[], const char *cmd_type,
                              const char *pc_name)
 {