LU-250 lnet: Router random shuffling support

This is the last of the router improvements that where
developed at ORNL. This feature allows routes to be
randomly placed. Our results show a 20 percent
improvement with random shuffling.

Signed-off-by: James Simmons <uja.ornl@gmail.com>
Change-Id: I6862febebd6797fb35c003646f6d90f5d5d2b014
Reviewed-on: http://review.whamcloud.com/249
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Isaac Huang <Isaac_Huang@xyratex.com>
Reviewed-by: Liang Zhen <liang@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index 932591c..fb726e3 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1452,6 +1452,9 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg)
                  * fairness; everything else being equal... */
                 cfs_list_del(&best_route->lr_list);
                 cfs_list_add_tail(&best_route->lr_list, &rnet->lrn_routes);
                  * fairness; everything else being equal... */
                 cfs_list_del(&best_route->lr_list);
                 cfs_list_add_tail(&best_route->lr_list, &rnet->lrn_routes);

+                CDEBUG(D_NET, "Best route to %s via %s for %s %d\n",
+                       libcfs_nid2str(dst_nid), libcfs_nid2str(lp->lp_nid),
+                       lnet_msgtyp2str(msg->msg_type), msg->msg_len);

 
                 if (src_ni == NULL) {
                         src_ni = lp->lp_ni;
 
                 if (src_ni == NULL) {
                         src_ni = lp->lp_ni;

diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c
index c82ff8e..6f23db0 100644 (file)
--- a/lnet/lnet/router.c
+++ b/lnet/lnet/router.c
@@ -235,6 +235,35 @@ lnet_find_net_locked (__u32 net)
         return NULL;
 }
 
         return NULL;
 }
 

+static void lnet_shuffle_seed(void)
+{
+        static int seeded = 0;
+        int lnd_type, seed[2];
+        struct timeval tv;
+        lnet_ni_t *ni;
+        cfs_list_t *tmp;
+
+        if (seeded)
+                return;
+
+        cfs_get_random_bytes(seed, sizeof(seed));
+
+        /* Nodes with small feet have little entropy
+         * the NID for this node gives the most entropy in the low bits */
+        cfs_list_for_each(tmp, &the_lnet.ln_nis) {
+                ni = cfs_list_entry(tmp, lnet_ni_t, ni_list);
+                lnd_type = LNET_NETTYP(LNET_NIDNET(ni->ni_nid));
+
+                if (lnd_type != LOLND)
+                        seed[0] ^= (LNET_NIDADDR(ni->ni_nid) | lnd_type);
+        }
+
+        cfs_gettimeofday(&tv);
+        cfs_srand(tv.tv_sec ^ seed[0], tv.tv_usec ^ seed[1]);
+        seeded = 1;
+        return;
+}
+

 /* NB expects LNET_LOCK held */
 void
 lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route)
 /* NB expects LNET_LOCK held */
 void
 lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route)


@@ -242,16 +271,15 @@ lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route)
         unsigned int      len = 0;
         unsigned int      offset = 0;
         cfs_list_t       *e;
         unsigned int      len = 0;
         unsigned int      offset = 0;
         cfs_list_t       *e;

-        extern __u64 lnet_create_interface_cookie(void);
+
+        lnet_shuffle_seed();

 
         cfs_list_for_each (e, &rnet->lrn_routes) {
                 len++;
         }
 
 
         cfs_list_for_each (e, &rnet->lrn_routes) {
                 len++;
         }
 

-        /* FIXME use Lustre random function when it's moved to libcfs.
-         * See bug 18751 */

         /* len+1 positions to add a new entry, also prevents division by 0 */
         /* len+1 positions to add a new entry, also prevents division by 0 */

-        offset = ((unsigned int) lnet_create_interface_cookie()) % (len + 1);
+        offset = cfs_rand() % (len + 1);

         cfs_list_for_each (e, &rnet->lrn_routes) {
                 if (offset == 0)
                         break;
         cfs_list_for_each (e, &rnet->lrn_routes) {
                 if (offset == 0)
                         break;