[tcpnal]: workaround: serialize access to procbridge pipe. TianYing's work

exposed the race condition here.

diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c
index 6da3210..dfcd743 100644 (file)
--- a/lnet/ulnds/procapi.c
+++ b/lnet/ulnds/procapi.c
@@ -48,6 +48,11 @@
  * forwards a packaged api call from the 'api' side to the 'library'
  *   side, and collects the result
  */
+/* XXX CFS workaround:
+ * when multiple threads call forward at the same time, data in the
+ * pipe will be trampled. add a mutex to serialize the access, as
+ * a temporary solution.
+ */
 #define forward_failure(operand,fd,buffer,length)\
        if(syscall(SYS_##operand,fd,buffer,length)!=length){\
           lib_fini(b->nal_cb);\
@@ -59,8 +64,12 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
     bridge b=(bridge)n->nal_data;
     procbridge p=(procbridge)b->local;
     int lib=p->to_lib[1];
+    static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
     int k;
 
+    /* protect the whole access to pipe */
+    pthread_mutex_lock(&mut);
+
     forward_failure(write,lib, &id, sizeof(id));
     forward_failure(write,lib,&args_len, sizeof(args_len));
     forward_failure(write,lib,&ret_len, sizeof(ret_len));
@@ -70,6 +79,8 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
         k=syscall(SYS_read, p->from_lib[0], ret, ret_len);
     } while ((k!=ret_len) && (errno += EINTR));
 
+    pthread_mutex_unlock(&mut);
+
     if(k!=ret_len){
         perror("nal: read return block");
         return PTL_SEGV;

diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c
index 6da3210..dfcd743 100644 (file)
--- a/lnet/ulnds/socklnd/procapi.c
+++ b/lnet/ulnds/socklnd/procapi.c
@@ -48,6 +48,11 @@
  * forwards a packaged api call from the 'api' side to the 'library'
  *   side, and collects the result
  */
+/* XXX CFS workaround:
+ * when multiple threads call forward at the same time, data in the
+ * pipe will be trampled. add a mutex to serialize the access, as
+ * a temporary solution.
+ */
 #define forward_failure(operand,fd,buffer,length)\
        if(syscall(SYS_##operand,fd,buffer,length)!=length){\
           lib_fini(b->nal_cb);\
@@ -59,8 +64,12 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
     bridge b=(bridge)n->nal_data;
     procbridge p=(procbridge)b->local;
     int lib=p->to_lib[1];
+    static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
     int k;
 
+    /* protect the whole access to pipe */
+    pthread_mutex_lock(&mut);
+
     forward_failure(write,lib, &id, sizeof(id));
     forward_failure(write,lib,&args_len, sizeof(args_len));
     forward_failure(write,lib,&ret_len, sizeof(ret_len));
@@ -70,6 +79,8 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
         k=syscall(SYS_read, p->from_lib[0], ret, ret_len);
     } while ((k!=ret_len) && (errno += EINTR));
 
+    pthread_mutex_unlock(&mut);
+
     if(k!=ret_len){
         perror("nal: read return block");
         return PTL_SEGV;

diff --git a/lustre/portals/unals/procapi.c b/lustre/portals/unals/procapi.c
index 6da3210..dfcd743 100644 (file)
--- a/lustre/portals/unals/procapi.c
+++ b/lustre/portals/unals/procapi.c
@@ -48,6 +48,11 @@
  * forwards a packaged api call from the 'api' side to the 'library'
  *   side, and collects the result
  */
+/* XXX CFS workaround:
+ * when multiple threads call forward at the same time, data in the
+ * pipe will be trampled. add a mutex to serialize the access, as
+ * a temporary solution.
+ */
 #define forward_failure(operand,fd,buffer,length)\
        if(syscall(SYS_##operand,fd,buffer,length)!=length){\
           lib_fini(b->nal_cb);\
@@ -59,8 +64,12 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
     bridge b=(bridge)n->nal_data;
     procbridge p=(procbridge)b->local;
     int lib=p->to_lib[1];
+    static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
     int k;
 
+    /* protect the whole access to pipe */
+    pthread_mutex_lock(&mut);
+
     forward_failure(write,lib, &id, sizeof(id));
     forward_failure(write,lib,&args_len, sizeof(args_len));
     forward_failure(write,lib,&ret_len, sizeof(ret_len));
@@ -70,6 +79,8 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len,
         k=syscall(SYS_read, p->from_lib[0], ret, ret_len);
     } while ((k!=ret_len) && (errno += EINTR));
 
+    pthread_mutex_unlock(&mut);
+
     if(k!=ret_len){
         perror("nal: read return block");
         return PTL_SEGV;