tcpnal:

  nal thread will call select() to block itself waiting incoming packets,
  but in 2 cases (1. new connection created 2. shutdown tcpnal) upper
  thread need wake up nal thread from sleep immediately.
  here we use a local socket which will be under select's monitoring to
  notify nal thread wakeup. again brings in unclean code into tcpnal.

diff --git a/lnet/ulnds/bridge.h b/lnet/ulnds/bridge.h
index 0b4940f..9a90ab8 100644 (file)
--- a/lnet/ulnds/bridge.h
+++ b/lnet/ulnds/bridge.h
@@ -6,6 +6,9 @@
  *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
  */
 
+#ifndef TCPNAL_PROCBRIDGE_H
+#define TCPNAL_PROCBRIDGE_H
+
 #include <portals/lib-p30.h>
 
 typedef struct bridge {
@@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal,
 
 typedef int (*nal_initialize)(bridge);
 extern nal_initialize nal_table[PTL_IFACE_MAX];
+
+#endif

diff --git a/lnet/ulnds/connection.c b/lnet/ulnds/connection.c
index 3e64b33..8b5a1c8 100644 (file)
--- a/lnet/ulnds/connection.c
+++ b/lnet/ulnds/connection.c
@@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
  */
 connection force_tcp_connection(manager m,
                                 unsigned int ip,
-                                unsigned short port)
+                                unsigned short port,
+                                procbridge pb)
 {
     connection conn;
     struct sockaddr_in addr;
@@ -357,6 +358,10 @@ connection force_tcp_connection(manager m,
             exit(-1);
 
         conn = allocate_connection(m, ip, port, fd);
+
+        /* let nal thread know this event right away */
+        if (conn)
+                procbridge_wakeup_nal(pb);
     }
 
     pthread_mutex_unlock(&m->conn_lock);

diff --git a/lnet/ulnds/connection.h b/lnet/ulnds/connection.h
index fb1eaab..343ffa6 100644 (file)
--- a/lnet/ulnds/connection.h
+++ b/lnet/ulnds/connection.h
@@ -7,6 +7,7 @@
  */
 
 #include <table.h>
+#include <procbridge.h>
 
 typedef struct manager {
     table connections;
@@ -26,7 +27,8 @@ typedef struct connection {
     manager m;
 } *connection;
 
-connection force_tcp_connection(manager m, unsigned int ip, unsigned int short);
+connection force_tcp_connection(manager m, unsigned int ip, unsigned int short,
+                                procbridge pb);
 manager init_connections(unsigned short, int (*f)(void *, void *), void *);
 void remove_connection(void *arg);
 void shutdown_connections(manager m);

diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c
index d1a445f..bddfe9a 100644 (file)
--- a/lnet/ulnds/procapi.c
+++ b/lnet/ulnds/procapi.c
@@ -32,12 +32,34 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
+#ifndef __CYGWIN__
+#include <syscall.h>
+#endif
+#include <sys/socket.h>
 #include <procbridge.h>
 #include <pqtimer.h>
 #include <dispatch.h>
 #include <errno.h>
 
 
+/* XXX CFS workaround, to give a chance to let nal thread wake up
+ * from waiting in select
+ */
+static int procbridge_notifier_handler(void *arg)
+{
+    static char buf[8];
+    procbridge p = (procbridge) arg;
+
+    syscall(SYS_read, p->notifier[1], buf, sizeof(buf));
+    return 1;
+}
+
+void procbridge_wakeup_nal(procbridge p)
+{
+    static char buf[8];
+    syscall(SYS_write, p->notifier[0], buf, sizeof(buf));
+}
+
 /* Function: forward
  * Arguments: nal_t *nal: pointer to my top-side nal structure
  *            id: the command to pass to the lower layer
@@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni)
     procbridge p=(procbridge)b->local;
 
     p->nal_flags |= NAL_FLAG_STOPPING;
+    procbridge_wakeup_nal(p);
 
     do {
         pthread_mutex_lock(&p->mutex);
@@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface,
     p->nal_flags = 0;
     pthread_mutex_init(&p->nal_cb_lock, 0);
 
+    /* initialize notifier */
+    if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) {
+        perror("socketpair failed");
+        return NULL;
+    }
+
+    if (!register_io_handler(p->notifier[1], READ_HANDLER,
+                procbridge_notifier_handler, p)) {
+        perror("fail to register notifier handler");
+        return NULL;
+    }
+
+    /* create nal thread */
     if (pthread_create(&p->t, NULL, nal_thread, &args)) {
         perror("nal_init: pthread_create");
         return(NULL);

diff --git a/lnet/ulnds/procbridge.h b/lnet/ulnds/procbridge.h
index 317e22f..965f83d 100644 (file)
--- a/lnet/ulnds/procbridge.h
+++ b/lnet/ulnds/procbridge.h
@@ -25,6 +25,9 @@ typedef struct procbridge {
     pthread_cond_t cond;
     pthread_mutex_t mutex;
 
+    /* socket pair used to notify nal thread */
+    int notifier[2];
+
     int nal_flags;
 
     pthread_mutex_t nal_cb_lock;
@@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface,
                                    ptl_pt_index_t ptl_size,
                                    ptl_ac_index_t acl_size,
                                    ptl_pid_t requested_pid);
+extern void procbridge_wakeup_nal(procbridge p);
 
 #endif

diff --git a/lnet/ulnds/socklnd/bridge.h b/lnet/ulnds/socklnd/bridge.h
index 0b4940f..9a90ab8 100644 (file)
--- a/lnet/ulnds/socklnd/bridge.h
+++ b/lnet/ulnds/socklnd/bridge.h
@@ -6,6 +6,9 @@
  *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
  */
 
+#ifndef TCPNAL_PROCBRIDGE_H
+#define TCPNAL_PROCBRIDGE_H
+
 #include <portals/lib-p30.h>
 
 typedef struct bridge {
@@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal,
 
 typedef int (*nal_initialize)(bridge);
 extern nal_initialize nal_table[PTL_IFACE_MAX];
+
+#endif

diff --git a/lnet/ulnds/socklnd/connection.c b/lnet/ulnds/socklnd/connection.c
index 3e64b33..8b5a1c8 100644 (file)
--- a/lnet/ulnds/socklnd/connection.c
+++ b/lnet/ulnds/socklnd/connection.c
@@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
  */
 connection force_tcp_connection(manager m,
                                 unsigned int ip,
-                                unsigned short port)
+                                unsigned short port,
+                                procbridge pb)
 {
     connection conn;
     struct sockaddr_in addr;
@@ -357,6 +358,10 @@ connection force_tcp_connection(manager m,
             exit(-1);
 
         conn = allocate_connection(m, ip, port, fd);
+
+        /* let nal thread know this event right away */
+        if (conn)
+                procbridge_wakeup_nal(pb);
     }
 
     pthread_mutex_unlock(&m->conn_lock);

diff --git a/lnet/ulnds/socklnd/connection.h b/lnet/ulnds/socklnd/connection.h
index fb1eaab..343ffa6 100644 (file)
--- a/lnet/ulnds/socklnd/connection.h
+++ b/lnet/ulnds/socklnd/connection.h
@@ -7,6 +7,7 @@
  */
 
 #include <table.h>
+#include <procbridge.h>
 
 typedef struct manager {
     table connections;
@@ -26,7 +27,8 @@ typedef struct connection {
     manager m;
 } *connection;
 
-connection force_tcp_connection(manager m, unsigned int ip, unsigned int short);
+connection force_tcp_connection(manager m, unsigned int ip, unsigned int short,
+                                procbridge pb);
 manager init_connections(unsigned short, int (*f)(void *, void *), void *);
 void remove_connection(void *arg);
 void shutdown_connections(manager m);

diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c
index d1a445f..bddfe9a 100644 (file)
--- a/lnet/ulnds/socklnd/procapi.c
+++ b/lnet/ulnds/socklnd/procapi.c
@@ -32,12 +32,34 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
+#ifndef __CYGWIN__
+#include <syscall.h>
+#endif
+#include <sys/socket.h>
 #include <procbridge.h>
 #include <pqtimer.h>
 #include <dispatch.h>
 #include <errno.h>
 
 
+/* XXX CFS workaround, to give a chance to let nal thread wake up
+ * from waiting in select
+ */
+static int procbridge_notifier_handler(void *arg)
+{
+    static char buf[8];
+    procbridge p = (procbridge) arg;
+
+    syscall(SYS_read, p->notifier[1], buf, sizeof(buf));
+    return 1;
+}
+
+void procbridge_wakeup_nal(procbridge p)
+{
+    static char buf[8];
+    syscall(SYS_write, p->notifier[0], buf, sizeof(buf));
+}
+
 /* Function: forward
  * Arguments: nal_t *nal: pointer to my top-side nal structure
  *            id: the command to pass to the lower layer
@@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni)
     procbridge p=(procbridge)b->local;
 
     p->nal_flags |= NAL_FLAG_STOPPING;
+    procbridge_wakeup_nal(p);
 
     do {
         pthread_mutex_lock(&p->mutex);
@@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface,
     p->nal_flags = 0;
     pthread_mutex_init(&p->nal_cb_lock, 0);
 
+    /* initialize notifier */
+    if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) {
+        perror("socketpair failed");
+        return NULL;
+    }
+
+    if (!register_io_handler(p->notifier[1], READ_HANDLER,
+                procbridge_notifier_handler, p)) {
+        perror("fail to register notifier handler");
+        return NULL;
+    }
+
+    /* create nal thread */
     if (pthread_create(&p->t, NULL, nal_thread, &args)) {
         perror("nal_init: pthread_create");
         return(NULL);

diff --git a/lnet/ulnds/socklnd/procbridge.h b/lnet/ulnds/socklnd/procbridge.h
index 317e22f..965f83d 100644 (file)
--- a/lnet/ulnds/socklnd/procbridge.h
+++ b/lnet/ulnds/socklnd/procbridge.h
@@ -25,6 +25,9 @@ typedef struct procbridge {
     pthread_cond_t cond;
     pthread_mutex_t mutex;
 
+    /* socket pair used to notify nal thread */
+    int notifier[2];
+
     int nal_flags;
 
     pthread_mutex_t nal_cb_lock;
@@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface,
                                    ptl_pt_index_t ptl_size,
                                    ptl_ac_index_t acl_size,
                                    ptl_pid_t requested_pid);
+extern void procbridge_wakeup_nal(procbridge p);
 
 #endif

diff --git a/lnet/ulnds/socklnd/tcplnd.c b/lnet/ulnds/socklnd/tcplnd.c
index 1041d1d..bf28a05 100644 (file)
--- a/lnet/ulnds/socklnd/tcplnd.c
+++ b/lnet/ulnds/socklnd/tcplnd.c
@@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n,
 
     if (!(c=force_tcp_connection((manager)b->lower,
                                  PNAL_IP(nid,b),
-                                 PNAL_PORT(nid,pid)))) 
+                                 PNAL_PORT(nid,pid),
+                                 b->local))) 
         return(1);
 
 #if 0

diff --git a/lnet/ulnds/tcplnd.c b/lnet/ulnds/tcplnd.c
index 1041d1d..bf28a05 100644 (file)
--- a/lnet/ulnds/tcplnd.c
+++ b/lnet/ulnds/tcplnd.c
@@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n,
 
     if (!(c=force_tcp_connection((manager)b->lower,
                                  PNAL_IP(nid,b),
-                                 PNAL_PORT(nid,pid)))) 
+                                 PNAL_PORT(nid,pid),
+                                 b->local))) 
         return(1);
 
 #if 0

diff --git a/lustre/portals/unals/bridge.h b/lustre/portals/unals/bridge.h
index 0b4940f..9a90ab8 100644 (file)
--- a/lustre/portals/unals/bridge.h
+++ b/lustre/portals/unals/bridge.h
@@ -6,6 +6,9 @@
  *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
  */
 
+#ifndef TCPNAL_PROCBRIDGE_H
+#define TCPNAL_PROCBRIDGE_H
+
 #include <portals/lib-p30.h>
 
 typedef struct bridge {
@@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal,
 
 typedef int (*nal_initialize)(bridge);
 extern nal_initialize nal_table[PTL_IFACE_MAX];
+
+#endif

diff --git a/lustre/portals/unals/connection.c b/lustre/portals/unals/connection.c
index 3e64b33..8b5a1c8 100644 (file)
--- a/lustre/portals/unals/connection.c
+++ b/lustre/portals/unals/connection.c
@@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
  */
 connection force_tcp_connection(manager m,
                                 unsigned int ip,
-                                unsigned short port)
+                                unsigned short port,
+                                procbridge pb)
 {
     connection conn;
     struct sockaddr_in addr;
@@ -357,6 +358,10 @@ connection force_tcp_connection(manager m,
             exit(-1);
 
         conn = allocate_connection(m, ip, port, fd);
+
+        /* let nal thread know this event right away */
+        if (conn)
+                procbridge_wakeup_nal(pb);
     }
 
     pthread_mutex_unlock(&m->conn_lock);

diff --git a/lustre/portals/unals/connection.h b/lustre/portals/unals/connection.h
index fb1eaab..343ffa6 100644 (file)
--- a/lustre/portals/unals/connection.h
+++ b/lustre/portals/unals/connection.h
@@ -7,6 +7,7 @@
  */
 
 #include <table.h>
+#include <procbridge.h>
 
 typedef struct manager {
     table connections;
@@ -26,7 +27,8 @@ typedef struct connection {
     manager m;
 } *connection;
 
-connection force_tcp_connection(manager m, unsigned int ip, unsigned int short);
+connection force_tcp_connection(manager m, unsigned int ip, unsigned int short,
+                                procbridge pb);
 manager init_connections(unsigned short, int (*f)(void *, void *), void *);
 void remove_connection(void *arg);
 void shutdown_connections(manager m);

diff --git a/lustre/portals/unals/procapi.c b/lustre/portals/unals/procapi.c
index d1a445f..bddfe9a 100644 (file)
--- a/lustre/portals/unals/procapi.c
+++ b/lustre/portals/unals/procapi.c
@@ -32,12 +32,34 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
+#ifndef __CYGWIN__
+#include <syscall.h>
+#endif
+#include <sys/socket.h>
 #include <procbridge.h>
 #include <pqtimer.h>
 #include <dispatch.h>
 #include <errno.h>
 
 
+/* XXX CFS workaround, to give a chance to let nal thread wake up
+ * from waiting in select
+ */
+static int procbridge_notifier_handler(void *arg)
+{
+    static char buf[8];
+    procbridge p = (procbridge) arg;
+
+    syscall(SYS_read, p->notifier[1], buf, sizeof(buf));
+    return 1;
+}
+
+void procbridge_wakeup_nal(procbridge p)
+{
+    static char buf[8];
+    syscall(SYS_write, p->notifier[0], buf, sizeof(buf));
+}
+
 /* Function: forward
  * Arguments: nal_t *nal: pointer to my top-side nal structure
  *            id: the command to pass to the lower layer
@@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni)
     procbridge p=(procbridge)b->local;
 
     p->nal_flags |= NAL_FLAG_STOPPING;
+    procbridge_wakeup_nal(p);
 
     do {
         pthread_mutex_lock(&p->mutex);
@@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface,
     p->nal_flags = 0;
     pthread_mutex_init(&p->nal_cb_lock, 0);
 
+    /* initialize notifier */
+    if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) {
+        perror("socketpair failed");
+        return NULL;
+    }
+
+    if (!register_io_handler(p->notifier[1], READ_HANDLER,
+                procbridge_notifier_handler, p)) {
+        perror("fail to register notifier handler");
+        return NULL;
+    }
+
+    /* create nal thread */
     if (pthread_create(&p->t, NULL, nal_thread, &args)) {
         perror("nal_init: pthread_create");
         return(NULL);

diff --git a/lustre/portals/unals/procbridge.h b/lustre/portals/unals/procbridge.h
index 317e22f..965f83d 100644 (file)
--- a/lustre/portals/unals/procbridge.h
+++ b/lustre/portals/unals/procbridge.h
@@ -25,6 +25,9 @@ typedef struct procbridge {
     pthread_cond_t cond;
     pthread_mutex_t mutex;
 
+    /* socket pair used to notify nal thread */
+    int notifier[2];
+
     int nal_flags;
 
     pthread_mutex_t nal_cb_lock;
@@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface,
                                    ptl_pt_index_t ptl_size,
                                    ptl_ac_index_t acl_size,
                                    ptl_pid_t requested_pid);
+extern void procbridge_wakeup_nal(procbridge p);
 
 #endif

diff --git a/lustre/portals/unals/tcpnal.c b/lustre/portals/unals/tcpnal.c
index 1041d1d..bf28a05 100644 (file)
--- a/lustre/portals/unals/tcpnal.c
+++ b/lustre/portals/unals/tcpnal.c
@@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n,
 
     if (!(c=force_tcp_connection((manager)b->lower,
                                  PNAL_IP(nid,b),
-                                 PNAL_PORT(nid,pid)))) 
+                                 PNAL_PORT(nid,pid),
+                                 b->local))) 
         return(1);
 
 #if 0