* changed router to do "obvious" routing automatically if enabled

*   tested routing between elan and tcp networks

diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c
index c8c435c..f0c2793 100644 (file)
--- a/lnet/klnds/qswlnd/qswlnd_cb.c
+++ b/lnet/klnds/qswlnd/qswlnd_cb.c
@@ -1513,8 +1513,8 @@ kqswnal_parse (kqswnal_rx_t *krx)
         LASSERTF (0, "checksums for forwarded packets not implemented\n");
 #endif
 
-        if (kqswnal_nid2elanid (dest_nid) >= 0)  /* should have gone direct to peer */
-        {
+        if (PTL_NIDNET(dest_nid) == PTL_NIDNET(kqswnal_data.kqn_ni->ni_nid)) {
+                /* should have gone direct to peer */
                 CERROR("dropping packet from %s for %s: target is peer\n", 
                        libcfs_nid2str(le64_to_cpu(hdr->src_nid)), 
                        libcfs_nid2str(dest_nid));
@@ -1942,7 +1942,7 @@ kqswnal_scheduler (void *arg)
                         spin_unlock_irqrestore (&kqswnal_data.kqn_sched_lock, flags);
 
                         /* If we're shutting down, this will just requeue fwd on kqn_idletxd_fwdq */
-                        kqswnal_fwd_packet (NULL, fwd);
+                        kqswnal_fwd_packet (kqswnal_data.kqn_ni, fwd);
 
                         did_something = 1;
                         spin_lock_irqsave (&kqswnal_data.kqn_sched_lock, flags);

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index 7a1be2b..e9fb82a 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -1156,13 +1156,15 @@ ksocknal_fwd_parse (ksock_conn_t *conn)
                 return;
         }
 
-        if (body_len > PTL_MTU) {      /* too big to forward */
+        if (PTL_NIDNET(conn->ksnc_hdr.dest_nid) ==
+            PTL_NIDNET(ksocknal_data.ksnd_ni->ni_nid)) {
+                /* should have gone direct */
                 CERROR ("dropping packet from %s for %s: "
-                        "packet size %d too big\n",
+                        "target is a peer\n",
                         libcfs_nid2str(src_nid),
-                        libcfs_nid2str(dest_nid),
-                        body_len);
-                /* on to new packet (skip this one's body) */
+                        libcfs_nid2str(dest_nid));
+
+                /* on to next packet (skip this one's body) */
                 ksocknal_new_packet (conn, body_len);
                 return;
         }
@@ -1176,17 +1178,14 @@ ksocknal_fwd_parse (ksock_conn_t *conn)
                 ksocknal_new_packet (conn, body_len); /* on to new packet */
                 return;
         }
-        
-        /* should have gone direct */
-        peer = ksocknal_find_peer (conn->ksnc_hdr.dest_nid);
-        if (peer != NULL) {
+
+        if (body_len > PTL_MTU) {      /* too big to forward */
                 CERROR ("dropping packet from %s for %s: "
-                        "target is a peer\n",
+                        "packet size %d too big\n",
                         libcfs_nid2str(src_nid),
-                        libcfs_nid2str(dest_nid));
-                ksocknal_peer_decref(peer);  /* drop ref from get above */
-
-                /* on to next packet (skip this one's body) */
+                        libcfs_nid2str(dest_nid),
+                        body_len);
+                /* on to new packet (skip this one's body) */
                 ksocknal_new_packet (conn, body_len);
                 return;
         }

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index d2c6549..bc4ef60 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -27,10 +27,6 @@ static char *networks = DEFAULT_NETWORKS;
 CFS_MODULE_PARM(networks, "s", charp, 0444,
                 "local networks (default='"DEFAULT_NETWORKS"')");
 
-static int nal_load_timeout = 10;
-CFS_MODULE_PARM(nal_load_timeout, "i", int, 0444,
-                "seconds to wait for a NAL to load");
-
 ptl_apini_t       ptl_apini;                    /* THE network interface (at the API) */
 
 void ptl_assert_wire_constants (void)

diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c
index 9943b3e..10057b9 100644 (file)
--- a/lnet/lnet/module.c
+++ b/lnet/lnet/module.c
@@ -25,6 +25,10 @@
 #define DEBUG_SUBSYSTEM S_PORTALS
 #include <portals/lib-p30.h>
 
+static int config_on_load = 1;
+CFS_MODULE_PARM(config_on_load, "i", int, 0444,
+                "configure network at module load");
+
 static int kportal_ioctl(unsigned int cmd, struct portal_ioctl_data *data)
 {
         ptl_err_t          initrc;
@@ -50,7 +54,7 @@ static int kportal_ioctl(unsigned int cmd, struct portal_ioctl_data *data)
         initrc = PtlNIInit(PTL_IFACE_DEFAULT, LUSTRE_SRV_PTL_PID, 
                            NULL, NULL, &nih);
         if (!(initrc == PTL_OK || initrc == PTL_IFACE_DUP))
-                RETURN (-EINVAL);
+                RETURN (-ENETDOWN);
 
         rc = PtlNICtl(nih, cmd, data);
 
@@ -79,6 +83,21 @@ static int init_kportals_module(void)
                 RETURN(rc);
         }
 
+        if (config_on_load) {
+                ptl_handle_ni_t    nih;
+
+                PTL_MUTEX_DOWN(&ptl_apini.apini_api_mutex);
+                ptl_apini.apini_niinit_self = 1;
+                PTL_MUTEX_UP(&ptl_apini.apini_api_mutex);
+
+                rc = PtlNIInit(PTL_IFACE_DEFAULT, LUSTRE_SRV_PTL_PID,
+                               NULL, NULL, &nih);
+                if (rc != PTL_OK) {
+                        PtlFini();
+                        return -ENETDOWN;
+                }
+        }
+        
         rc = libcfs_register_ioctl(&kportal_ioctl_handler);
         LASSERT (rc == 0);
 

diff --git a/lnet/router/proc.c b/lnet/router/proc.c
index 9bec9c4..5358492 100644 (file)
--- a/lnet/router/proc.c
+++ b/lnet/router/proc.c
@@ -24,7 +24,7 @@
 #include "router.h"
 #include <linux/seq_file.h>
 
-#define KPR_PROC_ROUTER "sys/portals/router"
+#define KPR_PROC_STATS  "sys/portals/router_stats"
 #define KPR_PROC_ROUTES "sys/portals/routes"
 
 static int 
@@ -239,10 +239,10 @@ kpr_proc_init(void)
         struct proc_dir_entry *stats;
         struct proc_dir_entry *routes;
 
-        /* Initialize KPR_PROC_ROUTER */
-        stats = create_proc_entry (KPR_PROC_ROUTER, 0644, NULL);
+        /* Initialize KPR_PROC_STATS */
+        stats = create_proc_entry (KPR_PROC_STATS, 0644, NULL);
         if (stats == NULL) {
-                CERROR("couldn't create proc entry %s\n", KPR_PROC_ROUTER);
+                CERROR("couldn't create proc entry %s\n", KPR_PROC_STATS);
                 return;
         }
 
@@ -264,6 +264,6 @@ kpr_proc_init(void)
 void
 kpr_proc_fini(void)
 {
-        remove_proc_entry(KPR_PROC_ROUTER, 0);
+        remove_proc_entry(KPR_PROC_STATS, 0);
         remove_proc_entry(KPR_PROC_ROUTES, 0);
 }

diff --git a/lnet/router/router.c b/lnet/router/router.c
index fff6c1a..47bb078 100644 (file)
--- a/lnet/router/router.c
+++ b/lnet/router/router.c
@@ -243,9 +243,6 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob)
        int                  rc = -ENOENT;
         __u32                target_net = PTL_NIDNET(target_nid);
         
-        /* Caller wants to know if 'target_nid' can be reached via a gateway
-         * ON HER OWN NETWORK */
-
         CDEBUG (D_NET, "lookup "LPX64" from %s\n", target_nid, 
                 (ni == NULL) ? "<>" : libcfs_nid2str(ni->ni_nid));
 
@@ -256,10 +253,15 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob)
                         return gwni->ni_nid;
                 }
         } else if (target_net == PTL_NIDNET(ni->ni_nid)) {
-                ptl_ni_addref(ni);    /* extra ref so caller can drop blindly */
+                /* Caller wants to know if 'target_nid' can be reached via a
+                 * gateway ON HER OWN NETWORK */
+                ptl_ni_addref(ni); /* extra ref so caller can drop blindly */
                 return ni->ni_nid;
         }
 
+        CDEBUG(D_NET, "%s from %s\n", libcfs_nid2str(target_nid),
+               (ni == NULL) ? "<none>" : libcfs_nid2str(ni->ni_nid));
+
        read_lock_irqsave(&kpr_state.kpr_rwlock, flags);
 
         if (ni != NULL && ni->ni_shutdown) {
@@ -283,22 +285,37 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob)
                         if (PTL_NIDNET(ni->ni_nid) != /* gateway not on ni's net */
                             PTL_NIDNET(re->kpre_gateway->kpge_nid))
                                 continue;
-                        tmpni = NULL;
+
+                        if (ge != NULL &&
+                            kpr_ge_isbetter (ge, re->kpre_gateway))
+                                continue;
+
+                } else if (gwni != NULL &&
+                           PTL_NIDNET(gwni->ni_nid) ==
+                           PTL_NIDNET(ge->kpge_nid)) {
+                        /* another gateway on the same net */
+
+                        if (kpr_ge_isbetter(ge, re->kpre_gateway))
+                                continue;
                 } else {
-                        tmpni = ptl_net2ni(PTL_NIDNET(ge->kpge_nid));
+                        /* another gateway on a new/different net */
+
+                        tmpni = ptl_net2ni(PTL_NIDNET(re->kpre_gateway->kpge_nid));
                         if (tmpni == NULL)      /* gateway not on a local net */
                                 continue;
-                }
                 
-                if (ge == NULL ||
-                    kpr_ge_isbetter (re->kpre_gateway, ge)) {
+                        if (ge != NULL &&
+                            kpr_ge_isbetter(ge, re->kpre_gateway)) {
+                                ptl_ni_decref(tmpni);
+                                continue;
+                        }
+                        
                         if (gwni != NULL)
                                 ptl_ni_decref(gwni);
-                        ge = re->kpre_gateway;
                         gwni = tmpni;
-                } else if (tmpni != NULL) {
-                        ptl_ni_decref(tmpni);
                 }
+
+                ge = re->kpre_gateway;
        }
 
         if (ge == NULL) {
@@ -308,7 +325,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob)
                 return PTL_NID_ANY;
         }
         
-        kpr_update_weight (ge, nob);
+        kpr_update_weight(ge, nob);
         gwnid = ge->kpge_nid;
        read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags);
         
@@ -335,8 +352,8 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd)
         __u32                target_net = PTL_NIDNET(target_nid);
         __u32                source_net = PTL_NIDNET(src_ni->ni_nid);
         int                  nob = fwd->kprfd_nob;
-        kpr_gateway_entry_t *ge = NULL;
-        ptl_ni_t            *dst_ni = NULL;
+        kpr_gateway_entry_t *ge;
+        ptl_ni_t            *dst_ni;
         ptl_ni_t            *tmp_ni;
         unsigned long        flags;
        struct list_head    *e;
@@ -346,6 +363,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd)
         CDEBUG (D_NET, "forward [%p] %s from %s\n", fwd,
                 libcfs_nid2str(target_nid), libcfs_nid2str(src_ni->ni_nid));
 
+        LASSERT (target_net != source_net);
         LASSERT (nob == ptl_kiov_nob (fwd->kprfd_niov, fwd->kprfd_kiov));
 
        fwd->kprfd_src_ni = src_ni;             /* stash calling ni */
@@ -368,8 +386,32 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd)
                goto out;
         }
 
-       /* Search routes for one that has a gateway to target_nid NOT on the caller's network */
+        /* Is the target_nid on a local network? */
+        dst_ni = ptl_net2ni(target_net);
+        if (dst_ni != NULL) {
+                if (dst_ni->ni_nal->nal_fwd == NULL) {
+                        rc = -EHOSTUNREACH;
+                        goto out;
+                }
+
+                fwd->kprfd_gateway_nid = dst_ni->ni_nid;
+                atomic_inc (&kpr_state.kpr_queue_depth);
+
+                read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags);
+
+                CDEBUG (D_NET, "forward [%p] %s: src ni %s dst ni %s\n",
+                        fwd, libcfs_nid2str(target_nid),
+                        libcfs_nid2str(src_ni->ni_nid),
+                        libcfs_nid2str(dst_ni->ni_nid));
 
+                dst_ni->ni_nal->nal_fwd(dst_ni, fwd);
+                ptl_ni_decref(dst_ni);
+                return;
+        }
+        
+       /* Search routes for one that has a gateway to target_nid NOT on the caller's network */
+        dst_ni = NULL;
+        ge = NULL;
         list_for_each (e, &kpr_state.kpr_routes) {
                re = list_entry (e, kpr_route_entry_t, kpre_list);
 
@@ -391,14 +433,17 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd)
                         continue;
                 }
                 
-                if (ge == NULL ||
-                    kpr_ge_isbetter (re->kpre_gateway, ge)) {
-                        if (dst_ni != NULL)
-                                ptl_ni_decref(dst_ni);
-                                
-                        dst_ni = tmp_ni;
-                        ge = re->kpre_gateway;
+                if (ge != NULL &&
+                    kpr_ge_isbetter(ge, re->kpre_gateway)) {
+                        ptl_ni_decref(tmp_ni);
+                        continue;
                 }
+                
+                if (dst_ni != NULL)
+                        ptl_ni_decref(dst_ni);
+                                
+                dst_ni = tmp_ni;
+                ge = re->kpre_gateway;
         }
         
         if (ge != NULL) {