LU-2934 lnet: Add LNet Router Priority parameter

This change adds a priority parameter to the route module settings.
This paramter can be >= 0. Like hops, the lower the prioirty number
the higher the priority.  So lower numbered priorities will be
selected over higher numbers.

Signed-off-by: Doug Oucharek <doug.s.oucharek@intel.com>
Change-Id: Idca12ee44ec5e2a896f89c178334363ffdfef4b8
Reviewed-on: http://review.whamcloud.com/5663
Tested-by: Hudson
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Amir Shehata <amir.shehata@intel.com>
Reviewed-by: Isaac Huang <he.huang@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/libcfs/include/libcfs/libcfs_ioctl.h b/libcfs/include/libcfs/libcfs_ioctl.h
index 31c76ec..dcbae24 100644 (file)
--- a/libcfs/include/libcfs/libcfs_ioctl.h
+++ b/libcfs/include/libcfs/libcfs_ioctl.h
@@ -69,6 +69,8 @@ struct libcfs_ioctl_data {
         char ioc_bulk[0];
 };
 
         char ioc_bulk[0];
 };
 

+#define ioc_priority ioc_u32[0]
+

 
 struct libcfs_ioctl_hdr {
         __u32 ioc_len;
 
 struct libcfs_ioctl_hdr {
         __u32 ioc_len;

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 24d7aba..8f9710e 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -726,12 +726,13 @@ extern lnet_ni_t *lnet_net2ni(__u32 net);
 
 int lnet_notify(lnet_ni_t *ni, lnet_nid_t peer, int alive, cfs_time_t when);
 void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, cfs_time_t when);
 
 int lnet_notify(lnet_ni_t *ni, lnet_nid_t peer, int alive, cfs_time_t when);
 void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, cfs_time_t when);

-int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid);
+int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid,
+                  unsigned int priority);

 int lnet_check_routes(void);
 int lnet_del_route(__u32 net, lnet_nid_t gw_nid);
 void lnet_destroy_routes(void);
 int lnet_get_route(int idx, __u32 *net, __u32 *hops,
 int lnet_check_routes(void);
 int lnet_del_route(__u32 net, lnet_nid_t gw_nid);
 void lnet_destroy_routes(void);
 int lnet_get_route(int idx, __u32 *net, __u32 *hops,

-                   lnet_nid_t *gateway, __u32 *alive);
+                  lnet_nid_t *gateway, __u32 *alive, __u32 *priority);

 void lnet_proc_init(void);
 void lnet_proc_fini(void);
 int  lnet_rtrpools_alloc(int im_a_router);
 void lnet_proc_init(void);
 void lnet_proc_fini(void);
 int  lnet_rtrpools_alloc(int im_a_router);

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index bfaa288..8aaa239 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -505,7 +505,6 @@ typedef struct lnet_peer {
        lnet_rc_data_t          *lp_rcd;        /* router checker state */
 } lnet_peer_t;
 
        lnet_rc_data_t          *lp_rcd;        /* router checker state */
 } lnet_peer_t;
 

-

 /* peer hash size */
 #define LNET_PEER_HASH_BITS     9
 #define LNET_PEER_HASH_SIZE     (1 << LNET_PEER_HASH_BITS)
 /* peer hash size */
 #define LNET_PEER_HASH_BITS     9
 #define LNET_PEER_HASH_SIZE     (1 << LNET_PEER_HASH_BITS)


@@ -531,6 +530,7 @@ typedef struct {
        int                     lr_seq;         /* sequence for round-robin */
        unsigned int            lr_downis;      /* number of down NIs */
        unsigned int            lr_hops;        /* how far I am */
        int                     lr_seq;         /* sequence for round-robin */
        unsigned int            lr_downis;      /* number of down NIs */
        unsigned int            lr_hops;        /* how far I am */

+       unsigned int            lr_priority;    /* route priority */

 } lnet_route_t;
 
 #define LNET_REMOTE_NETS_HASH_DEFAULT  (1U << 7)
 } lnet_route_t;
 
 #define LNET_REMOTE_NETS_HASH_DEFAULT  (1U << 7)

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 97bb5e7..ed5eba9 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -1599,8 +1599,8 @@ LNetCtl(unsigned int cmd, void *arg)
                 return lnet_fail_nid(data->ioc_nid, data->ioc_count);
 
         case IOC_LIBCFS_ADD_ROUTE:
                 return lnet_fail_nid(data->ioc_nid, data->ioc_count);
 
         case IOC_LIBCFS_ADD_ROUTE:

-                rc = lnet_add_route(data->ioc_net, data->ioc_count,
-                                    data->ioc_nid);
+               rc = lnet_add_route(data->ioc_net, data->ioc_count,
+                                   data->ioc_nid, data->ioc_priority);

                 return (rc != 0) ? rc : lnet_check_routes();
 
         case IOC_LIBCFS_DEL_ROUTE:
                 return (rc != 0) ? rc : lnet_check_routes();
 
         case IOC_LIBCFS_DEL_ROUTE:


@@ -1609,7 +1609,8 @@ LNetCtl(unsigned int cmd, void *arg)
         case IOC_LIBCFS_GET_ROUTE:
                 return lnet_get_route(data->ioc_count,
                                       &data->ioc_net, &data->ioc_count,
         case IOC_LIBCFS_GET_ROUTE:
                 return lnet_get_route(data->ioc_count,
                                       &data->ioc_net, &data->ioc_count,

-                                      &data->ioc_nid, &data->ioc_flags);
+                                     &data->ioc_nid, &data->ioc_flags,
+                                     &data->ioc_priority);

         case IOC_LIBCFS_NOTIFY_ROUTER:
                 return lnet_notify(NULL, data->ioc_nid, data->ioc_flags,
                                    cfs_time_current() -
         case IOC_LIBCFS_NOTIFY_ROUTER:
                 return lnet_notify(NULL, data->ioc_nid, data->ioc_flags,
                                    cfs_time_current() -

diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c
index b18c103..deca792 100644 (file)
--- a/lnet/lnet/config.c
+++ b/lnet/lnet/config.c
@@ -614,6 +614,37 @@ lnet_parse_hops (char *str, unsigned int *hops)
                 *hops > 0 && *hops < 256);
 }
 
                 *hops > 0 && *hops < 256);
 }
 

+#define LNET_PRIORITY_SEPARATOR (':')
+
+int
+lnet_parse_priority(char *str, unsigned int *priority, char **token)
+{
+       int   nob;
+       char *sep;
+       int   len;
+
+       sep = strchr(str, LNET_PRIORITY_SEPARATOR);
+       if (sep == NULL) {
+               *priority = 0;
+               return 0;
+       }
+       len = strlen(sep + 1);
+
+       if ((sscanf((sep+1), "%u%n", priority, &nob) < 1) || (len != nob)) {
+               /* Update the caller's token pointer so it treats the found
+                  priority as the token to report in the error message. */
+               *token += sep - str + 1;
+               return -1;
+       }
+
+       CDEBUG(D_NET, "gateway %s, priority %d, nob %d\n", str, *priority, nob);
+
+       /*
+        * Change priority separator to \0 to be able to parse NID
+        */
+       *sep = '\0';
+       return 0;
+}

 
 int
 lnet_parse_route (char *str, int *im_a_router)
 
 int
 lnet_parse_route (char *str, int *im_a_router)


@@ -635,6 +666,7 @@ lnet_parse_route (char *str, int *im_a_router)
         int               myrc = -1;
         unsigned int      hops;
         int               got_hops = 0;
         int               myrc = -1;
         unsigned int      hops;
         int               got_hops = 0;

+       unsigned int      priority = 0;

 
        CFS_INIT_LIST_HEAD(&gateways);
        CFS_INIT_LIST_HEAD(&nets);
 
        CFS_INIT_LIST_HEAD(&gateways);
        CFS_INIT_LIST_HEAD(&nets);


@@ -702,6 +734,11 @@ lnet_parse_route (char *str, int *im_a_router)
                                     LNET_NETTYP(net) == LOLND)
                                        goto token_error;
                        } else {
                                     LNET_NETTYP(net) == LOLND)
                                        goto token_error;
                        } else {

+                               rc = lnet_parse_priority(ltb->ltb_text,
+                                                        &priority, &token);
+                               if (rc < 0)
+                                       goto token_error;
+

                                nid = libcfs_str2nid(ltb->ltb_text);
                                if (nid == LNET_NID_ANY ||
                                     LNET_NETTYP(LNET_NIDNET(nid)) == LOLND)
                                nid = libcfs_str2nid(ltb->ltb_text);
                                if (nid == LNET_NID_ANY ||
                                     LNET_NETTYP(LNET_NIDNET(nid)) == LOLND)


@@ -724,14 +761,14 @@ lnet_parse_route (char *str, int *im_a_router)
                cfs_list_for_each (tmp2, &gateways) {
                        ltb = cfs_list_entry(tmp2, lnet_text_buf_t, ltb_list);
                        nid = libcfs_str2nid(ltb->ltb_text);
                cfs_list_for_each (tmp2, &gateways) {
                        ltb = cfs_list_entry(tmp2, lnet_text_buf_t, ltb_list);
                        nid = libcfs_str2nid(ltb->ltb_text);

-                       LASSERT (nid != LNET_NID_ANY);
+                       LASSERT(nid != LNET_NID_ANY);

 
                         if (lnet_islocalnid(nid)) {
                                 *im_a_router = 1;
                                 continue;
                         }
                         
 
                         if (lnet_islocalnid(nid)) {
                                 *im_a_router = 1;
                                 continue;
                         }
                         

-                        rc = lnet_add_route (net, hops, nid);
+                       rc = lnet_add_route(net, hops, nid, priority);

                         if (rc != 0) {
                                 CERROR("Can't create route "
                                        "to %s via %s\n",
                         if (rc != 0) {
                                 CERROR("Can't create route "
                                        "to %s via %s\n",

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index a531d94..0906690 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1122,6 +1122,12 @@ lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2)
        lnet_peer_t *p1 = r1->lr_gateway;
        lnet_peer_t *p2 = r2->lr_gateway;
 
        lnet_peer_t *p1 = r1->lr_gateway;
        lnet_peer_t *p2 = r2->lr_gateway;
 

+       if (r1->lr_priority < r2->lr_priority)
+               return 1;
+
+       if (r1->lr_priority > r2->lr_priority)
+               return -1;
+

        if (r1->lr_hops < r2->lr_hops)
                return 1;
 
        if (r1->lr_hops < r2->lr_hops)
                return 1;
 

diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c
index dd5e9e3..46adbeb 100644 (file)
--- a/lnet/lnet/router.c
+++ b/lnet/lnet/router.c
@@ -301,7 +301,8 @@ lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route)
 }
 
 int
 }
 
 int

-lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway)
+lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway,
+              unsigned int priority)

 {
         cfs_list_t          *e;
         lnet_remotenet_t    *rnet;
 {
         cfs_list_t          *e;
         lnet_remotenet_t    *rnet;


@@ -311,8 +312,8 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway)
         int                  add_route;
         int                  rc;
 
         int                  add_route;
         int                  rc;
 

-        CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n",
-               libcfs_net2str(net), hops, libcfs_nid2str(gateway));
+       CDEBUG(D_NET, "Add route: net %s hops %u priority %u gw %s\n",
+              libcfs_net2str(net), hops, priority, libcfs_nid2str(gateway));

 
         if (gateway == LNET_NID_ANY ||
             LNET_NETTYP(LNET_NIDNET(gateway)) == LOLND ||
 
         if (gateway == LNET_NID_ANY ||
             LNET_NETTYP(LNET_NIDNET(gateway)) == LOLND ||


@@ -342,6 +343,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway)
         rnet->lrn_net = net;
         route->lr_hops = hops;
        route->lr_net = net;
         rnet->lrn_net = net;
         route->lr_hops = hops;
        route->lr_net = net;

+       route->lr_priority = priority;

 
        lnet_net_lock(LNET_LOCK_EX);
 
 
        lnet_net_lock(LNET_LOCK_EX);
 


@@ -552,7 +554,7 @@ lnet_destroy_routes (void)
 
 int
 lnet_get_route(int idx, __u32 *net, __u32 *hops,
 
 int
 lnet_get_route(int idx, __u32 *net, __u32 *hops,

-              lnet_nid_t *gateway, __u32 *alive)
+              lnet_nid_t *gateway, __u32 *alive, __u32 *priority)

 {
        cfs_list_t              *e1;
        cfs_list_t              *e2;
 {
        cfs_list_t              *e1;
        cfs_list_t              *e2;


@@ -574,10 +576,11 @@ lnet_get_route(int idx, __u32 *net, __u32 *hops,
                                                       lr_list);
 
                                if (idx-- == 0) {
                                                       lr_list);
 
                                if (idx-- == 0) {

-                                       *net     = rnet->lrn_net;
-                                       *hops    = route->lr_hops;
-                                       *gateway = route->lr_gateway->lp_nid;
-                                       *alive   = route->lr_gateway->lp_alive;
+                                       *net      = rnet->lrn_net;
+                                       *hops     = route->lr_hops;
+                                       *priority = route->lr_priority;
+                                       *gateway  = route->lr_gateway->lp_nid;
+                                       *alive    = route->lr_gateway->lp_alive;

                                        lnet_net_unlock(cpt);
                                        return 0;
                                }
                                        lnet_net_unlock(cpt);
                                        return 0;
                                }

diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c
index d4382b0..6bb052b 100644 (file)
--- a/lnet/lnet/router_proc.c
+++ b/lnet/lnet/router_proc.c
@@ -185,8 +185,8 @@ int LL_PROC_PROTO(proc_lnet_routes)
                               the_lnet.ln_routing ? "enabled" : "disabled");
                 LASSERT (tmpstr + tmpsiz - s > 0);
 
                               the_lnet.ln_routing ? "enabled" : "disabled");
                 LASSERT (tmpstr + tmpsiz - s > 0);
 

-                s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %7s %s\n",
-                              "net", "hops", "state", "router");
+               s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %8s %7s %s\n",
+                             "net", "hops", "priority", "state", "router");

                 LASSERT (tmpstr + tmpsiz - s > 0);
 
                lnet_net_lock(0);
                 LASSERT (tmpstr + tmpsiz - s > 0);
 
                lnet_net_lock(0);


@@ -240,14 +240,16 @@ int LL_PROC_PROTO(proc_lnet_routes)
                }
 
                 if (route != NULL) {
                }
 
                 if (route != NULL) {

-                        __u32        net   = rnet->lrn_net;
-                        unsigned int hops  = route->lr_hops;
-                        lnet_nid_t   nid   = route->lr_gateway->lp_nid;
-                        int          alive = route->lr_gateway->lp_alive;
+                       __u32        net        = rnet->lrn_net;
+                       unsigned int hops       = route->lr_hops;
+                       unsigned int priority   = route->lr_priority;
+                       lnet_nid_t   nid        = route->lr_gateway->lp_nid;
+                       int          alive      = route->lr_gateway->lp_alive;

 
                        s += snprintf(s, tmpstr + tmpsiz - s,
 
                        s += snprintf(s, tmpstr + tmpsiz - s,

-                                     "%-8s %4u %7s %s\n",
+                                     "%-8s %4u %8u %7s %s\n",

                                      libcfs_net2str(net), hops,
                                      libcfs_net2str(net), hops,

+                                     priority,

                                      alive ? "up" : "down",
                                      libcfs_nid2str(nid));
                        LASSERT(tmpstr + tmpsiz - s > 0);
                                      alive ? "up" : "down",
                                      libcfs_nid2str(nid));
                        LASSERT(tmpstr + tmpsiz - s > 0);

diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c
index 3c09a8d..2b3aada 100644 (file)
--- a/lnet/utils/portals.c
+++ b/lnet/utils/portals.c
@@ -1126,14 +1126,15 @@ jt_ptl_add_route (int argc, char **argv)
         struct libcfs_ioctl_data data;
         lnet_nid_t               gateway_nid;
         unsigned int             hops = 1;
         struct libcfs_ioctl_data data;
         lnet_nid_t               gateway_nid;
         unsigned int             hops = 1;

+       unsigned int             priority = 0;

         char                    *end;
         int                      rc;
 
         char                    *end;
         int                      rc;
 

-        if (argc < 2 || argc > 3)
-        {
-                fprintf (stderr, "usage: %s gateway [hopcount]\n", argv[0]);
-                return (0);
-        }
+       if (argc < 2 || argc > 4) {
+               fprintf(stderr, "usage: %s gateway [hopcount [priority]]\n",
+                       argv[0]);
+               return -1;
+       }

 
         if (!g_net_is_set(argv[0]))
                 return (-1);
 
         if (!g_net_is_set(argv[0]))
                 return (-1);


@@ -1144,18 +1145,29 @@ jt_ptl_add_route (int argc, char **argv)
                 return (-1);
         }
 
                 return (-1);
         }
 

-        if (argc == 3) {
-                hops = strtoul(argv[2], &end, 0);
-                if (hops >= 256 || *end != 0) {
-                        fprintf (stderr, "Can't parse hopcount \"%s\"\n", argv[2]);
-                        return -1;
-                }
-        }
-
-        LIBCFS_IOC_INIT(data);
-        data.ioc_net = g_net;
-        data.ioc_count = hops;
-        data.ioc_nid = gateway_nid;
+       if (argc > 2) {
+               hops = strtoul(argv[2], &end, 0);
+               if (hops == 0 || hops >= 256 || (end != NULL && *end != 0)) {
+                       fprintf(stderr, "Can't parse hopcount \"%s\"\n",
+                               argv[2]);
+                       return -1;
+               }
+               if (argc == 4) {
+                       priority = strtoul(argv[3], &end, 0);
+                       if (end != NULL && *end != 0) {
+                               fprintf(stderr,
+                                       "Can't parse priority \"%s\"\n",
+                                       argv[3]);
+                               return -1;
+                       }
+               }
+       }
+
+       LIBCFS_IOC_INIT(data);
+       data.ioc_net = g_net;
+       data.ioc_count = hops;
+       data.ioc_nid = gateway_nid;
+       data.ioc_priority = priority;

 
         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_ROUTE, &data);
         if (rc != 0) {
 
         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_ROUTE, &data);
         if (rc != 0) {


@@ -1267,6 +1279,7 @@ jt_ptl_print_routes (int argc, char **argv)
         lnet_nid_t                nid;
         unsigned int              hops;
         int                       alive;
         lnet_nid_t                nid;
         unsigned int              hops;
         int                       alive;

+       unsigned int              pri;

 
         for (index = 0;;index++)
         {
 
         for (index = 0;;index++)
         {


@@ -1281,10 +1294,11 @@ jt_ptl_print_routes (int argc, char **argv)
                 hops    = data.ioc_count;
                 nid     = data.ioc_nid;
                 alive   = data.ioc_flags;
                 hops    = data.ioc_count;
                 nid     = data.ioc_nid;
                 alive   = data.ioc_flags;

+               pri     = data.ioc_priority;

 
 

-                printf ("net %18s hops %u gw %32s %s\n", 
-                        libcfs_net2str(net), hops,
-                        libcfs_nid2str(nid), alive ? "up" : "down");
+               printf("net %18s hops %u gw %32s %s pri %u\n",
+                      libcfs_net2str(net), hops,
+                      libcfs_nid2str(nid), alive ? "up" : "down", pri);

         }
 
         if (errno != ENOENT)
         }
 
         if (errno != ENOENT)

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 2b653bc..fcf5f72 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -10679,12 +10679,13 @@ test_215() { # for bugs 18102, 21079, 21517
 
        # /proc/sys/lnet/routes should look like this:
        # Routing disabled/enabled
 
        # /proc/sys/lnet/routes should look like this:
        # Routing disabled/enabled

-       # net hops state router
-       # where net is a string like tcp0, hops >= 0, state is up/down,
+       # net hops priority state router
+       # where net is a string like tcp0, hops > 0, priority >= 0,
+       # state is up/down,

        # router is a string like 192.168.1.1@tcp2
        L1="^Routing (disabled|enabled)$"
        # router is a string like 192.168.1.1@tcp2
        L1="^Routing (disabled|enabled)$"

-       L2="^net +hops +state +router$"
-       BR="^$NET +$N +(up|down) +$NID$"
+       L2="^net +hops +priority +state +router$"
+       BR="^$NET +$N +(0|1) +(up|down) +$NID$"

        create_lnet_proc_files "routes"
        check_lnet_proc_entry "routes.out" "/proc/sys/lnet/routes" "$BR" "$L1" "$L2"
        check_lnet_proc_entry "routes.sys" "lnet.routes" "$BR" "$L1" "$L2"
        create_lnet_proc_files "routes"
        check_lnet_proc_entry "routes.out" "/proc/sys/lnet/routes" "$BR" "$L1" "$L2"
        check_lnet_proc_entry "routes.sys" "lnet.routes" "$BR" "$L1" "$L2"

diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c
index b86a378..4363223 100644 (file)
--- a/lustre/utils/lctl.c
+++ b/lustre/utils/lctl.c
@@ -402,7 +402,7 @@ command_t cmdlist[] = {
          "usage: del_interface [ip]"},
         {"add_route", jt_ptl_add_route, 0,
          "add an entry to the portals routing table\n"
          "usage: del_interface [ip]"},
         {"add_route", jt_ptl_add_route, 0,
          "add an entry to the portals routing table\n"

-         "usage: add_route <gateway> <target> [<target>]"},
+        "usage: add_route <gateway> [<hops> [<priority>]]"},

         {"del_route", jt_ptl_del_route, 0,
          "delete route via gateway to targets from the portals routing table\n"
          "usage: del_route <gateway> [<target>] [<target>]"},
         {"del_route", jt_ptl_del_route, 0,
          "delete route via gateway to targets from the portals routing table\n"
          "usage: del_route <gateway> [<target>] [<target>]"},