X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Fulnds%2Ftcplnd.c;h=0c47f426779e5e47bc54fdc696ec590c3b46f253;hp=d6b5ed08645f135276623ff6ab3ccf25790a8088;hb=ff17cc7e0282d9b1522810e0c5d12171c4d46a2d;hpb=c39da0c8b593d91ccf458a520ec2d0fd704a4bd9 diff --git a/lnet/ulnds/tcplnd.c b/lnet/ulnds/tcplnd.c index d6b5ed0..0c47f42 100644 --- a/lnet/ulnds/tcplnd.c +++ b/lnet/ulnds/tcplnd.c @@ -2,6 +2,7 @@ * vim:expandtab:shiftwidth=8:tabstop=8: * * Copyright (c) 2002 Cray Inc. + * Copyright (c) 2003 Cluster File Systems, Inc. * * This file is part of Lustre, http://www.lustre.org. * @@ -27,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +36,11 @@ #include #include #include +#include +#include +#ifndef __CYGWIN__ +#include +#endif /* Function: tcpnal_send * Arguments: nal: pointer to my nal control block @@ -50,55 +55,89 @@ * * sends a packet to the peer, after insuring that a connection exists */ -#warning FIXME: "param 'type' is newly added, make use of it!!" -int tcpnal_send(nal_cb_t *n, - void *private, - lib_msg_t *cookie, - ptl_hdr_t *hdr, - int type, - ptl_nid_t nid, - ptl_pid_t pid, - unsigned int niov, - struct iovec *iov, - size_t len) +ptl_err_t tcpnal_send(nal_cb_t *n, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t len) { connection c; bridge b=(bridge)n->nal_data; - struct iovec tiov[2]; - int count = 1; + struct iovec tiov[257]; + static pthread_mutex_t send_lock = PTHREAD_MUTEX_INITIALIZER; + ptl_err_t rc = PTL_OK; + int sysrc; + int total; + int ntiov; + int i; if (!(c=force_tcp_connection((manager)b->lower, PNAL_IP(nid,b), - PNAL_PORT(nid,pid)))) - return(1); + PNAL_PORT(nid,pid), + b->local))) + return(PTL_FAIL); -#if 0 /* TODO: these results should be checked. furthermore, provision must be made for the SIGPIPE which is delivered when writing on a tcp socket which has closed underneath the application. there is a linux flag in the sendmsg call which turns off the signally behaviour, but its nonstandard */ - syscall(SYS_write, c->fd,hdr,sizeof(ptl_hdr_t)); - LASSERT (niov <= 1); - if (len) syscall(SYS_write, c->fd,iov[0].iov_base,len); -#else - LASSERT (niov <= 1); + + LASSERT (niov <= 256); tiov[0].iov_base = hdr; tiov[0].iov_len = sizeof(ptl_hdr_t); + ntiov = 1 + lib_extract_iov(256, &tiov[1], niov, iov, offset, len); - if (len) { - tiov[1].iov_base = iov[0].iov_base; - tiov[1].iov_len = len; - count++; + pthread_mutex_lock(&send_lock); +#if 1 + for (i = total = 0; i < ntiov; i++) + total += tiov[i].iov_len; + + sysrc = syscall(SYS_writev, c->fd, tiov, ntiov); + if (sysrc != total) { + fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n", + rc, total, errno); + rc = PTL_FAIL; + } +#else + for (i = total = 0; i <= ntiov; i++) { + rc = send(c->fd, tiov[i].iov_base, tiov[i].iov_len, 0); + + if (rc != tiov[i].iov_len) { + fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n", + rc, tiov[i].iov_len, errno); + rc = PTL_FAIL; + break; + } + total += rc; } - - syscall(SYS_writev, c->fd, tiov, count); #endif - lib_finalize(n, private, cookie); - - return(0); +#if 0 + fprintf (stderr, "sent %s total %d in %d frags\n", + hdr->type == PTL_MSG_ACK ? "ACK" : + hdr->type == PTL_MSG_PUT ? "PUT" : + hdr->type == PTL_MSG_GET ? "GET" : + hdr->type == PTL_MSG_REPLY ? "REPLY" : + hdr->type == PTL_MSG_HELLO ? "HELLO" : "UNKNOWN", + total, niov + 1); +#endif + pthread_mutex_unlock(&send_lock); + + if (rc == PTL_OK) { + /* NB the NAL only calls lib_finalize() if it returns PTL_OK + * from cb_send() */ + lib_finalize(n, private, cookie, PTL_OK); + } + + return(rc); } @@ -115,20 +154,40 @@ int tcpnal_send(nal_cb_t *n, * blocking read of the requested data. must drain out the * difference of mainpulated and requested lengths from the network */ -int tcpnal_recv(nal_cb_t *n, - void *private, - lib_msg_t *cookie, - unsigned int niov, - struct iovec *iov, - ptl_size_t mlen, - ptl_size_t rlen) +ptl_err_t tcpnal_recv(nal_cb_t *n, + void *private, + lib_msg_t *cookie, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t mlen, + size_t rlen) { - if (mlen) { - LASSERT (niov <= 1); - read_connection(private,iov[0].iov_base,mlen); - lib_finalize(n, private, cookie); - } + struct iovec tiov[256]; + int ntiov; + int i; + + if (!niov) + goto finalize; + + LASSERT(mlen); + LASSERT(rlen); + LASSERT(rlen >= mlen); + + ntiov = lib_extract_iov(256, tiov, niov, iov, offset, mlen); + + /* FIXME + * 1. Is this effecient enough? change to use readv() directly? + * 2. need check return from read_connection() + * - MeiJia + */ + for (i = 0; i < ntiov; i++) + read_connection(private, tiov[i].iov_base, tiov[i].iov_len); + +finalize: + /* FIXME; we always assume success here... */ + lib_finalize(n, private, cookie, PTL_OK); if (mlen!=rlen){ char *trash=malloc(rlen-mlen); @@ -138,7 +197,7 @@ int tcpnal_recv(nal_cb_t *n, free(trash); } - return(rlen); + return(PTL_OK); } @@ -153,15 +212,15 @@ int tcpnal_recv(nal_cb_t *n, */ static int from_connection(void *a, void *d) { - connection c = d; - bridge b=a; - ptl_hdr_t hdr; - - if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ - lib_parse(b->nal_cb, &hdr, c); - return(1); - } - return(0); + connection c = d; + bridge b = a; + ptl_hdr_t hdr; + + if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ + lib_parse(b->nal_cb, &hdr, c); + return(1); + } + return(0); }