tcpnal: need exchange NID when tcp connection created, ksocknal now default

require this.

diff --git a/lnet/ulnds/connection.c b/lnet/ulnds/connection.c
index 4dfd4b1..6ba1c22 100644 (file)
--- a/lnet/ulnds/connection.c
+++ b/lnet/ulnds/connection.c
@@ -37,6 +37,11 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <portals/types.h>
+#include <portals/list.h>
+#include <portals/lib-types.h>
+#include <portals/socknal.h>
+#include <linux/kp30.h>
 #include <connection.h>
 #include <pthread.h>
 #include <errno.h>
@@ -44,7 +49,6 @@
 #include <syscall.h>
 #endif
 
-
 /* global variable: acceptor port */
 unsigned short tcpnal_acceptor_port = 988;
 
@@ -196,6 +200,105 @@ static int new_connection(void *z)
     return(1);
 }
 
+/* FIXME assuming little endian, cleanup!! */
+#define __cpu_to_le64(x) ((__u64)(x))
+#define __le64_to_cpu(x) ((__u64)(x))
+#define __cpu_to_le32(x) ((__u32)(x))
+#define __le32_to_cpu(x) ((__u32)(x))
+#define __cpu_to_le16(x) ((__u16)(x))
+#define __le16_to_cpu(x) ((__u16)(x))
+
+extern ptl_nid_t tcpnal_mynid;
+
+int
+tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
+{
+        int                 rc;
+        ptl_hdr_t           hdr;
+        ptl_magicversion_t *hmv = (ptl_magicversion_t *)&hdr.dest_nid;
+
+        LASSERT (sizeof (*hmv) == sizeof (hdr.dest_nid));
+
+        memset (&hdr, 0, sizeof (hdr));
+        hmv->magic         = __cpu_to_le32 (PORTALS_PROTO_MAGIC);
+        hmv->version_major = __cpu_to_le32 (PORTALS_PROTO_VERSION_MAJOR);
+        hmv->version_minor = __cpu_to_le32 (PORTALS_PROTO_VERSION_MINOR);
+        
+        hdr.src_nid = __cpu_to_le64 (tcpnal_mynid);
+        hdr.type    = __cpu_to_le32 (PTL_MSG_HELLO);
+
+        hdr.msg.hello.type = __cpu_to_le32 (type);
+        hdr.msg.hello.incarnation = 0;
+
+        /* Assume sufficient socket buffering for this message */
+        rc = syscall(SYS_write, sockfd, &hdr, sizeof(hdr));
+        if (rc <= 0) {
+                CERROR ("Error %d sending HELLO to %llx\n", rc, *nid);
+                return (rc);
+        }
+
+        rc = syscall(SYS_read, sockfd, hmv, sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading HELLO from %llx\n", rc, *nid);
+                return (rc);
+        }
+        
+        if (hmv->magic != __le32_to_cpu (PORTALS_PROTO_MAGIC)) {
+                CERROR ("Bad magic %#08x (%#08x expected) from %llx\n",
+                        __cpu_to_le32 (hmv->magic), PORTALS_PROTO_MAGIC, *nid);
+                return (-EPROTO);
+        }
+
+        if (hmv->version_major != __cpu_to_le16 (PORTALS_PROTO_VERSION_MAJOR) ||
+            hmv->version_minor != __cpu_to_le16 (PORTALS_PROTO_VERSION_MINOR)) {
+                CERROR ("Incompatible protocol version %d.%d (%d.%d expected)"
+                        " from %llx\n",
+                        __le16_to_cpu (hmv->version_major),
+                        __le16_to_cpu (hmv->version_minor),
+                        PORTALS_PROTO_VERSION_MAJOR,
+                        PORTALS_PROTO_VERSION_MINOR,
+                        *nid);
+                return (-EPROTO);
+        }
+
+#if (PORTALS_PROTO_VERSION_MAJOR != 0)
+# error "This code only understands protocol version 0.x"
+#endif
+        /* version 0 sends magic/version as the dest_nid of a 'hello' header,
+         * so read the rest of it in now... */
+
+        rc = syscall(SYS_read, sockfd, hmv + 1, sizeof(hdr) - sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading rest of HELLO hdr from %llx\n",
+                        rc, *nid);
+                return (rc);
+        }
+
+        /* ...and check we got what we expected */
+        if (hdr.type != __cpu_to_le32 (PTL_MSG_HELLO) ||
+            hdr.payload_length != __cpu_to_le32 (0)) {
+                CERROR ("Expecting a HELLO hdr with 0 payload,"
+                        " but got type %d with %d payload from %llx\n",
+                        __le32_to_cpu (hdr.type),
+                        __le32_to_cpu (hdr.payload_length), *nid);
+                return (-EPROTO);
+        }
+
+        if (__le64_to_cpu(hdr.src_nid) == PTL_NID_ANY) {
+                CERROR("Expecting a HELLO hdr with a NID, but got PTL_NID_ANY\n");
+                return (-EPROTO);
+        }
+
+        if (*nid == PTL_NID_ANY) {              /* don't know peer's nid yet */
+                *nid = __le64_to_cpu(hdr.src_nid);
+        } else if (*nid != __le64_to_cpu (hdr.src_nid)) {
+                CERROR ("Connected to nid %llx, but expecting %llx\n",
+                        __le64_to_cpu (hdr.src_nid), *nid);
+                return (-EPROTO);
+        }
+
+        return (0);
+}
 
 /* Function:  force_tcp_connection
  * Arguments: t: tcpnal
@@ -221,6 +324,7 @@ connection force_tcp_connection(manager m,
     conn = hash_table_find(m->connections, id);
     if (!conn) {
         int fd;
+        ptl_nid_t peernid = PTL_NID_ANY;
 
         bzero((char *) &addr, sizeof(addr));
         addr.sin_family      = AF_INET;
@@ -236,6 +340,10 @@ connection force_tcp_connection(manager m,
             perror("tcpnal connect");
             return(0);
         }
+        /* say hello */
+        if (tcpnal_hello(fd, &peernid, SOCKNAL_CONN_ANY, 0))
+            exit(-1);
+
         conn = allocate_connection(m, ip, port, fd);
     }
 

diff --git a/lnet/ulnds/socklnd/connection.c b/lnet/ulnds/socklnd/connection.c
index 4dfd4b1..6ba1c22 100644 (file)
--- a/lnet/ulnds/socklnd/connection.c
+++ b/lnet/ulnds/socklnd/connection.c
@@ -37,6 +37,11 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <portals/types.h>
+#include <portals/list.h>
+#include <portals/lib-types.h>
+#include <portals/socknal.h>
+#include <linux/kp30.h>
 #include <connection.h>
 #include <pthread.h>
 #include <errno.h>
@@ -44,7 +49,6 @@
 #include <syscall.h>
 #endif
 
-
 /* global variable: acceptor port */
 unsigned short tcpnal_acceptor_port = 988;
 
@@ -196,6 +200,105 @@ static int new_connection(void *z)
     return(1);
 }
 
+/* FIXME assuming little endian, cleanup!! */
+#define __cpu_to_le64(x) ((__u64)(x))
+#define __le64_to_cpu(x) ((__u64)(x))
+#define __cpu_to_le32(x) ((__u32)(x))
+#define __le32_to_cpu(x) ((__u32)(x))
+#define __cpu_to_le16(x) ((__u16)(x))
+#define __le16_to_cpu(x) ((__u16)(x))
+
+extern ptl_nid_t tcpnal_mynid;
+
+int
+tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
+{
+        int                 rc;
+        ptl_hdr_t           hdr;
+        ptl_magicversion_t *hmv = (ptl_magicversion_t *)&hdr.dest_nid;
+
+        LASSERT (sizeof (*hmv) == sizeof (hdr.dest_nid));
+
+        memset (&hdr, 0, sizeof (hdr));
+        hmv->magic         = __cpu_to_le32 (PORTALS_PROTO_MAGIC);
+        hmv->version_major = __cpu_to_le32 (PORTALS_PROTO_VERSION_MAJOR);
+        hmv->version_minor = __cpu_to_le32 (PORTALS_PROTO_VERSION_MINOR);
+        
+        hdr.src_nid = __cpu_to_le64 (tcpnal_mynid);
+        hdr.type    = __cpu_to_le32 (PTL_MSG_HELLO);
+
+        hdr.msg.hello.type = __cpu_to_le32 (type);
+        hdr.msg.hello.incarnation = 0;
+
+        /* Assume sufficient socket buffering for this message */
+        rc = syscall(SYS_write, sockfd, &hdr, sizeof(hdr));
+        if (rc <= 0) {
+                CERROR ("Error %d sending HELLO to %llx\n", rc, *nid);
+                return (rc);
+        }
+
+        rc = syscall(SYS_read, sockfd, hmv, sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading HELLO from %llx\n", rc, *nid);
+                return (rc);
+        }
+        
+        if (hmv->magic != __le32_to_cpu (PORTALS_PROTO_MAGIC)) {
+                CERROR ("Bad magic %#08x (%#08x expected) from %llx\n",
+                        __cpu_to_le32 (hmv->magic), PORTALS_PROTO_MAGIC, *nid);
+                return (-EPROTO);
+        }
+
+        if (hmv->version_major != __cpu_to_le16 (PORTALS_PROTO_VERSION_MAJOR) ||
+            hmv->version_minor != __cpu_to_le16 (PORTALS_PROTO_VERSION_MINOR)) {
+                CERROR ("Incompatible protocol version %d.%d (%d.%d expected)"
+                        " from %llx\n",
+                        __le16_to_cpu (hmv->version_major),
+                        __le16_to_cpu (hmv->version_minor),
+                        PORTALS_PROTO_VERSION_MAJOR,
+                        PORTALS_PROTO_VERSION_MINOR,
+                        *nid);
+                return (-EPROTO);
+        }
+
+#if (PORTALS_PROTO_VERSION_MAJOR != 0)
+# error "This code only understands protocol version 0.x"
+#endif
+        /* version 0 sends magic/version as the dest_nid of a 'hello' header,
+         * so read the rest of it in now... */
+
+        rc = syscall(SYS_read, sockfd, hmv + 1, sizeof(hdr) - sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading rest of HELLO hdr from %llx\n",
+                        rc, *nid);
+                return (rc);
+        }
+
+        /* ...and check we got what we expected */
+        if (hdr.type != __cpu_to_le32 (PTL_MSG_HELLO) ||
+            hdr.payload_length != __cpu_to_le32 (0)) {
+                CERROR ("Expecting a HELLO hdr with 0 payload,"
+                        " but got type %d with %d payload from %llx\n",
+                        __le32_to_cpu (hdr.type),
+                        __le32_to_cpu (hdr.payload_length), *nid);
+                return (-EPROTO);
+        }
+
+        if (__le64_to_cpu(hdr.src_nid) == PTL_NID_ANY) {
+                CERROR("Expecting a HELLO hdr with a NID, but got PTL_NID_ANY\n");
+                return (-EPROTO);
+        }
+
+        if (*nid == PTL_NID_ANY) {              /* don't know peer's nid yet */
+                *nid = __le64_to_cpu(hdr.src_nid);
+        } else if (*nid != __le64_to_cpu (hdr.src_nid)) {
+                CERROR ("Connected to nid %llx, but expecting %llx\n",
+                        __le64_to_cpu (hdr.src_nid), *nid);
+                return (-EPROTO);
+        }
+
+        return (0);
+}
 
 /* Function:  force_tcp_connection
  * Arguments: t: tcpnal
@@ -221,6 +324,7 @@ connection force_tcp_connection(manager m,
     conn = hash_table_find(m->connections, id);
     if (!conn) {
         int fd;
+        ptl_nid_t peernid = PTL_NID_ANY;
 
         bzero((char *) &addr, sizeof(addr));
         addr.sin_family      = AF_INET;
@@ -236,6 +340,10 @@ connection force_tcp_connection(manager m,
             perror("tcpnal connect");
             return(0);
         }
+        /* say hello */
+        if (tcpnal_hello(fd, &peernid, SOCKNAL_CONN_ANY, 0))
+            exit(-1);
+
         conn = allocate_connection(m, ip, port, fd);
     }
 

diff --git a/lustre/portals/unals/connection.c b/lustre/portals/unals/connection.c
index 4dfd4b1..6ba1c22 100644 (file)
--- a/lustre/portals/unals/connection.c
+++ b/lustre/portals/unals/connection.c
@@ -37,6 +37,11 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <portals/types.h>
+#include <portals/list.h>
+#include <portals/lib-types.h>
+#include <portals/socknal.h>
+#include <linux/kp30.h>
 #include <connection.h>
 #include <pthread.h>
 #include <errno.h>
@@ -44,7 +49,6 @@
 #include <syscall.h>
 #endif
 
-
 /* global variable: acceptor port */
 unsigned short tcpnal_acceptor_port = 988;
 
@@ -196,6 +200,105 @@ static int new_connection(void *z)
     return(1);
 }
 
+/* FIXME assuming little endian, cleanup!! */
+#define __cpu_to_le64(x) ((__u64)(x))
+#define __le64_to_cpu(x) ((__u64)(x))
+#define __cpu_to_le32(x) ((__u32)(x))
+#define __le32_to_cpu(x) ((__u32)(x))
+#define __cpu_to_le16(x) ((__u16)(x))
+#define __le16_to_cpu(x) ((__u16)(x))
+
+extern ptl_nid_t tcpnal_mynid;
+
+int
+tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
+{
+        int                 rc;
+        ptl_hdr_t           hdr;
+        ptl_magicversion_t *hmv = (ptl_magicversion_t *)&hdr.dest_nid;
+
+        LASSERT (sizeof (*hmv) == sizeof (hdr.dest_nid));
+
+        memset (&hdr, 0, sizeof (hdr));
+        hmv->magic         = __cpu_to_le32 (PORTALS_PROTO_MAGIC);
+        hmv->version_major = __cpu_to_le32 (PORTALS_PROTO_VERSION_MAJOR);
+        hmv->version_minor = __cpu_to_le32 (PORTALS_PROTO_VERSION_MINOR);
+        
+        hdr.src_nid = __cpu_to_le64 (tcpnal_mynid);
+        hdr.type    = __cpu_to_le32 (PTL_MSG_HELLO);
+
+        hdr.msg.hello.type = __cpu_to_le32 (type);
+        hdr.msg.hello.incarnation = 0;
+
+        /* Assume sufficient socket buffering for this message */
+        rc = syscall(SYS_write, sockfd, &hdr, sizeof(hdr));
+        if (rc <= 0) {
+                CERROR ("Error %d sending HELLO to %llx\n", rc, *nid);
+                return (rc);
+        }
+
+        rc = syscall(SYS_read, sockfd, hmv, sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading HELLO from %llx\n", rc, *nid);
+                return (rc);
+        }
+        
+        if (hmv->magic != __le32_to_cpu (PORTALS_PROTO_MAGIC)) {
+                CERROR ("Bad magic %#08x (%#08x expected) from %llx\n",
+                        __cpu_to_le32 (hmv->magic), PORTALS_PROTO_MAGIC, *nid);
+                return (-EPROTO);
+        }
+
+        if (hmv->version_major != __cpu_to_le16 (PORTALS_PROTO_VERSION_MAJOR) ||
+            hmv->version_minor != __cpu_to_le16 (PORTALS_PROTO_VERSION_MINOR)) {
+                CERROR ("Incompatible protocol version %d.%d (%d.%d expected)"
+                        " from %llx\n",
+                        __le16_to_cpu (hmv->version_major),
+                        __le16_to_cpu (hmv->version_minor),
+                        PORTALS_PROTO_VERSION_MAJOR,
+                        PORTALS_PROTO_VERSION_MINOR,
+                        *nid);
+                return (-EPROTO);
+        }
+
+#if (PORTALS_PROTO_VERSION_MAJOR != 0)
+# error "This code only understands protocol version 0.x"
+#endif
+        /* version 0 sends magic/version as the dest_nid of a 'hello' header,
+         * so read the rest of it in now... */
+
+        rc = syscall(SYS_read, sockfd, hmv + 1, sizeof(hdr) - sizeof(*hmv));
+        if (rc <= 0) {
+                CERROR ("Error %d reading rest of HELLO hdr from %llx\n",
+                        rc, *nid);
+                return (rc);
+        }
+
+        /* ...and check we got what we expected */
+        if (hdr.type != __cpu_to_le32 (PTL_MSG_HELLO) ||
+            hdr.payload_length != __cpu_to_le32 (0)) {
+                CERROR ("Expecting a HELLO hdr with 0 payload,"
+                        " but got type %d with %d payload from %llx\n",
+                        __le32_to_cpu (hdr.type),
+                        __le32_to_cpu (hdr.payload_length), *nid);
+                return (-EPROTO);
+        }
+
+        if (__le64_to_cpu(hdr.src_nid) == PTL_NID_ANY) {
+                CERROR("Expecting a HELLO hdr with a NID, but got PTL_NID_ANY\n");
+                return (-EPROTO);
+        }
+
+        if (*nid == PTL_NID_ANY) {              /* don't know peer's nid yet */
+                *nid = __le64_to_cpu(hdr.src_nid);
+        } else if (*nid != __le64_to_cpu (hdr.src_nid)) {
+                CERROR ("Connected to nid %llx, but expecting %llx\n",
+                        __le64_to_cpu (hdr.src_nid), *nid);
+                return (-EPROTO);
+        }
+
+        return (0);
+}
 
 /* Function:  force_tcp_connection
  * Arguments: t: tcpnal
@@ -221,6 +324,7 @@ connection force_tcp_connection(manager m,
     conn = hash_table_find(m->connections, id);
     if (!conn) {
         int fd;
+        ptl_nid_t peernid = PTL_NID_ANY;
 
         bzero((char *) &addr, sizeof(addr));
         addr.sin_family      = AF_INET;
@@ -236,6 +340,10 @@ connection force_tcp_connection(manager m,
             perror("tcpnal connect");
             return(0);
         }
+        /* say hello */
+        if (tcpnal_hello(fd, &peernid, SOCKNAL_CONN_ANY, 0))
+            exit(-1);
+
         conn = allocate_connection(m, ip, port, fd);
     }