From 2060acacde0938575aa46a5d00b586817f253e94 Mon Sep 17 00:00:00 2001 From: ericm Date: Thu, 4 Dec 2003 05:55:53 +0000 Subject: [PATCH] tcpnal: nal thread will call select() to block itself waiting incoming packets, but in 2 cases (1. new connection created 2. shutdown tcpnal) upper thread need wake up nal thread from sleep immediately. here we use a local socket which will be under select's monitoring to notify nal thread wakeup. again brings in unclean code into tcpnal. --- lnet/ulnds/bridge.h | 5 +++++ lnet/ulnds/connection.c | 7 ++++++- lnet/ulnds/connection.h | 4 +++- lnet/ulnds/procapi.c | 36 ++++++++++++++++++++++++++++++++++++ lnet/ulnds/procbridge.h | 4 ++++ lnet/ulnds/socklnd/bridge.h | 5 +++++ lnet/ulnds/socklnd/connection.c | 7 ++++++- lnet/ulnds/socklnd/connection.h | 4 +++- lnet/ulnds/socklnd/procapi.c | 36 ++++++++++++++++++++++++++++++++++++ lnet/ulnds/socklnd/procbridge.h | 4 ++++ lnet/ulnds/socklnd/tcplnd.c | 3 ++- lnet/ulnds/tcplnd.c | 3 ++- lustre/portals/unals/bridge.h | 5 +++++ lustre/portals/unals/connection.c | 7 ++++++- lustre/portals/unals/connection.h | 4 +++- lustre/portals/unals/procapi.c | 36 ++++++++++++++++++++++++++++++++++++ lustre/portals/unals/procbridge.h | 4 ++++ lustre/portals/unals/tcpnal.c | 3 ++- 18 files changed, 168 insertions(+), 9 deletions(-) diff --git a/lnet/ulnds/bridge.h b/lnet/ulnds/bridge.h index 0b4940f..9a90ab8 100644 --- a/lnet/ulnds/bridge.h +++ b/lnet/ulnds/bridge.h @@ -6,6 +6,9 @@ * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ */ +#ifndef TCPNAL_PROCBRIDGE_H +#define TCPNAL_PROCBRIDGE_H + #include typedef struct bridge { @@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal, typedef int (*nal_initialize)(bridge); extern nal_initialize nal_table[PTL_IFACE_MAX]; + +#endif diff --git a/lnet/ulnds/connection.c b/lnet/ulnds/connection.c index 3e64b33..8b5a1c8 100644 --- a/lnet/ulnds/connection.c +++ b/lnet/ulnds/connection.c @@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation) */ connection force_tcp_connection(manager m, unsigned int ip, - unsigned short port) + unsigned short port, + procbridge pb) { connection conn; struct sockaddr_in addr; @@ -357,6 +358,10 @@ connection force_tcp_connection(manager m, exit(-1); conn = allocate_connection(m, ip, port, fd); + + /* let nal thread know this event right away */ + if (conn) + procbridge_wakeup_nal(pb); } pthread_mutex_unlock(&m->conn_lock); diff --git a/lnet/ulnds/connection.h b/lnet/ulnds/connection.h index fb1eaab..343ffa6 100644 --- a/lnet/ulnds/connection.h +++ b/lnet/ulnds/connection.h @@ -7,6 +7,7 @@ */ #include +#include typedef struct manager { table connections; @@ -26,7 +27,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short, + procbridge pb); manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c index d1a445f..bddfe9a 100644 --- a/lnet/ulnds/procapi.c +++ b/lnet/ulnds/procapi.c @@ -32,12 +32,34 @@ #include #include #include +#ifndef __CYGWIN__ +#include +#endif +#include #include #include #include #include +/* XXX CFS workaround, to give a chance to let nal thread wake up + * from waiting in select + */ +static int procbridge_notifier_handler(void *arg) +{ + static char buf[8]; + procbridge p = (procbridge) arg; + + syscall(SYS_read, p->notifier[1], buf, sizeof(buf)); + return 1; +} + +void procbridge_wakeup_nal(procbridge p) +{ + static char buf[8]; + syscall(SYS_write, p->notifier[0], buf, sizeof(buf)); +} + /* Function: forward * Arguments: nal_t *nal: pointer to my top-side nal structure * id: the command to pass to the lower layer @@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni) procbridge p=(procbridge)b->local; p->nal_flags |= NAL_FLAG_STOPPING; + procbridge_wakeup_nal(p); do { pthread_mutex_lock(&p->mutex); @@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface, p->nal_flags = 0; pthread_mutex_init(&p->nal_cb_lock, 0); + /* initialize notifier */ + if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) { + perror("socketpair failed"); + return NULL; + } + + if (!register_io_handler(p->notifier[1], READ_HANDLER, + procbridge_notifier_handler, p)) { + perror("fail to register notifier handler"); + return NULL; + } + + /* create nal thread */ if (pthread_create(&p->t, NULL, nal_thread, &args)) { perror("nal_init: pthread_create"); return(NULL); diff --git a/lnet/ulnds/procbridge.h b/lnet/ulnds/procbridge.h index 317e22f..965f83d 100644 --- a/lnet/ulnds/procbridge.h +++ b/lnet/ulnds/procbridge.h @@ -25,6 +25,9 @@ typedef struct procbridge { pthread_cond_t cond; pthread_mutex_t mutex; + /* socket pair used to notify nal thread */ + int notifier[2]; + int nal_flags; pthread_mutex_t nal_cb_lock; @@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface, ptl_pt_index_t ptl_size, ptl_ac_index_t acl_size, ptl_pid_t requested_pid); +extern void procbridge_wakeup_nal(procbridge p); #endif diff --git a/lnet/ulnds/socklnd/bridge.h b/lnet/ulnds/socklnd/bridge.h index 0b4940f..9a90ab8 100644 --- a/lnet/ulnds/socklnd/bridge.h +++ b/lnet/ulnds/socklnd/bridge.h @@ -6,6 +6,9 @@ * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ */ +#ifndef TCPNAL_PROCBRIDGE_H +#define TCPNAL_PROCBRIDGE_H + #include typedef struct bridge { @@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal, typedef int (*nal_initialize)(bridge); extern nal_initialize nal_table[PTL_IFACE_MAX]; + +#endif diff --git a/lnet/ulnds/socklnd/connection.c b/lnet/ulnds/socklnd/connection.c index 3e64b33..8b5a1c8 100644 --- a/lnet/ulnds/socklnd/connection.c +++ b/lnet/ulnds/socklnd/connection.c @@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation) */ connection force_tcp_connection(manager m, unsigned int ip, - unsigned short port) + unsigned short port, + procbridge pb) { connection conn; struct sockaddr_in addr; @@ -357,6 +358,10 @@ connection force_tcp_connection(manager m, exit(-1); conn = allocate_connection(m, ip, port, fd); + + /* let nal thread know this event right away */ + if (conn) + procbridge_wakeup_nal(pb); } pthread_mutex_unlock(&m->conn_lock); diff --git a/lnet/ulnds/socklnd/connection.h b/lnet/ulnds/socklnd/connection.h index fb1eaab..343ffa6 100644 --- a/lnet/ulnds/socklnd/connection.h +++ b/lnet/ulnds/socklnd/connection.h @@ -7,6 +7,7 @@ */ #include +#include typedef struct manager { table connections; @@ -26,7 +27,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short, + procbridge pb); manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c index d1a445f..bddfe9a 100644 --- a/lnet/ulnds/socklnd/procapi.c +++ b/lnet/ulnds/socklnd/procapi.c @@ -32,12 +32,34 @@ #include #include #include +#ifndef __CYGWIN__ +#include +#endif +#include #include #include #include #include +/* XXX CFS workaround, to give a chance to let nal thread wake up + * from waiting in select + */ +static int procbridge_notifier_handler(void *arg) +{ + static char buf[8]; + procbridge p = (procbridge) arg; + + syscall(SYS_read, p->notifier[1], buf, sizeof(buf)); + return 1; +} + +void procbridge_wakeup_nal(procbridge p) +{ + static char buf[8]; + syscall(SYS_write, p->notifier[0], buf, sizeof(buf)); +} + /* Function: forward * Arguments: nal_t *nal: pointer to my top-side nal structure * id: the command to pass to the lower layer @@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni) procbridge p=(procbridge)b->local; p->nal_flags |= NAL_FLAG_STOPPING; + procbridge_wakeup_nal(p); do { pthread_mutex_lock(&p->mutex); @@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface, p->nal_flags = 0; pthread_mutex_init(&p->nal_cb_lock, 0); + /* initialize notifier */ + if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) { + perror("socketpair failed"); + return NULL; + } + + if (!register_io_handler(p->notifier[1], READ_HANDLER, + procbridge_notifier_handler, p)) { + perror("fail to register notifier handler"); + return NULL; + } + + /* create nal thread */ if (pthread_create(&p->t, NULL, nal_thread, &args)) { perror("nal_init: pthread_create"); return(NULL); diff --git a/lnet/ulnds/socklnd/procbridge.h b/lnet/ulnds/socklnd/procbridge.h index 317e22f..965f83d 100644 --- a/lnet/ulnds/socklnd/procbridge.h +++ b/lnet/ulnds/socklnd/procbridge.h @@ -25,6 +25,9 @@ typedef struct procbridge { pthread_cond_t cond; pthread_mutex_t mutex; + /* socket pair used to notify nal thread */ + int notifier[2]; + int nal_flags; pthread_mutex_t nal_cb_lock; @@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface, ptl_pt_index_t ptl_size, ptl_ac_index_t acl_size, ptl_pid_t requested_pid); +extern void procbridge_wakeup_nal(procbridge p); #endif diff --git a/lnet/ulnds/socklnd/tcplnd.c b/lnet/ulnds/socklnd/tcplnd.c index 1041d1d..bf28a058 100644 --- a/lnet/ulnds/socklnd/tcplnd.c +++ b/lnet/ulnds/socklnd/tcplnd.c @@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n, if (!(c=force_tcp_connection((manager)b->lower, PNAL_IP(nid,b), - PNAL_PORT(nid,pid)))) + PNAL_PORT(nid,pid), + b->local))) return(1); #if 0 diff --git a/lnet/ulnds/tcplnd.c b/lnet/ulnds/tcplnd.c index 1041d1d..bf28a058 100644 --- a/lnet/ulnds/tcplnd.c +++ b/lnet/ulnds/tcplnd.c @@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n, if (!(c=force_tcp_connection((manager)b->lower, PNAL_IP(nid,b), - PNAL_PORT(nid,pid)))) + PNAL_PORT(nid,pid), + b->local))) return(1); #if 0 diff --git a/lustre/portals/unals/bridge.h b/lustre/portals/unals/bridge.h index 0b4940f..9a90ab8 100644 --- a/lustre/portals/unals/bridge.h +++ b/lustre/portals/unals/bridge.h @@ -6,6 +6,9 @@ * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ */ +#ifndef TCPNAL_PROCBRIDGE_H +#define TCPNAL_PROCBRIDGE_H + #include typedef struct bridge { @@ -27,3 +30,5 @@ nal_t *bridge_init(ptl_interface_t nal, typedef int (*nal_initialize)(bridge); extern nal_initialize nal_table[PTL_IFACE_MAX]; + +#endif diff --git a/lustre/portals/unals/connection.c b/lustre/portals/unals/connection.c index 3e64b33..8b5a1c8 100644 --- a/lustre/portals/unals/connection.c +++ b/lustre/portals/unals/connection.c @@ -309,7 +309,8 @@ tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation) */ connection force_tcp_connection(manager m, unsigned int ip, - unsigned short port) + unsigned short port, + procbridge pb) { connection conn; struct sockaddr_in addr; @@ -357,6 +358,10 @@ connection force_tcp_connection(manager m, exit(-1); conn = allocate_connection(m, ip, port, fd); + + /* let nal thread know this event right away */ + if (conn) + procbridge_wakeup_nal(pb); } pthread_mutex_unlock(&m->conn_lock); diff --git a/lustre/portals/unals/connection.h b/lustre/portals/unals/connection.h index fb1eaab..343ffa6 100644 --- a/lustre/portals/unals/connection.h +++ b/lustre/portals/unals/connection.h @@ -7,6 +7,7 @@ */ #include +#include typedef struct manager { table connections; @@ -26,7 +27,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short, + procbridge pb); manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); diff --git a/lustre/portals/unals/procapi.c b/lustre/portals/unals/procapi.c index d1a445f..bddfe9a 100644 --- a/lustre/portals/unals/procapi.c +++ b/lustre/portals/unals/procapi.c @@ -32,12 +32,34 @@ #include #include #include +#ifndef __CYGWIN__ +#include +#endif +#include #include #include #include #include +/* XXX CFS workaround, to give a chance to let nal thread wake up + * from waiting in select + */ +static int procbridge_notifier_handler(void *arg) +{ + static char buf[8]; + procbridge p = (procbridge) arg; + + syscall(SYS_read, p->notifier[1], buf, sizeof(buf)); + return 1; +} + +void procbridge_wakeup_nal(procbridge p) +{ + static char buf[8]; + syscall(SYS_write, p->notifier[0], buf, sizeof(buf)); +} + /* Function: forward * Arguments: nal_t *nal: pointer to my top-side nal structure * id: the command to pass to the lower layer @@ -79,6 +101,7 @@ static int procbridge_shutdown(nal_t *n, int ni) procbridge p=(procbridge)b->local; p->nal_flags |= NAL_FLAG_STOPPING; + procbridge_wakeup_nal(p); do { pthread_mutex_lock(&p->mutex); @@ -212,6 +235,19 @@ nal_t *procbridge_interface(int num_interface, p->nal_flags = 0; pthread_mutex_init(&p->nal_cb_lock, 0); + /* initialize notifier */ + if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) { + perror("socketpair failed"); + return NULL; + } + + if (!register_io_handler(p->notifier[1], READ_HANDLER, + procbridge_notifier_handler, p)) { + perror("fail to register notifier handler"); + return NULL; + } + + /* create nal thread */ if (pthread_create(&p->t, NULL, nal_thread, &args)) { perror("nal_init: pthread_create"); return(NULL); diff --git a/lustre/portals/unals/procbridge.h b/lustre/portals/unals/procbridge.h index 317e22f..965f83d 100644 --- a/lustre/portals/unals/procbridge.h +++ b/lustre/portals/unals/procbridge.h @@ -25,6 +25,9 @@ typedef struct procbridge { pthread_cond_t cond; pthread_mutex_t mutex; + /* socket pair used to notify nal thread */ + int notifier[2]; + int nal_flags; pthread_mutex_t nal_cb_lock; @@ -51,5 +54,6 @@ extern nal_t *procbridge_interface(int num_interface, ptl_pt_index_t ptl_size, ptl_ac_index_t acl_size, ptl_pid_t requested_pid); +extern void procbridge_wakeup_nal(procbridge p); #endif diff --git a/lustre/portals/unals/tcpnal.c b/lustre/portals/unals/tcpnal.c index 1041d1d..bf28a058 100644 --- a/lustre/portals/unals/tcpnal.c +++ b/lustre/portals/unals/tcpnal.c @@ -76,7 +76,8 @@ int tcpnal_send(nal_cb_t *n, if (!(c=force_tcp_connection((manager)b->lower, PNAL_IP(nid,b), - PNAL_PORT(nid,pid)))) + PNAL_PORT(nid,pid), + b->local))) return(1); #if 0 -- 1.8.3.1