1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2002 Cray Inc.
5 * Copyright (c) 2003 Cluster File Systems, Inc.
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 This file implements the TCP-based nal by providing glue
25 between the connection service and the generic NAL implementation */
31 #include <sys/types.h>
32 #include <sys/socket.h>
33 #include <netinet/in.h>
38 #include <connection.h>
45 /* Function: tcpnal_send
46 * Arguments: nal: pointer to my nal control block
48 * cookie: passed back to the portals library
49 * hdr: pointer to the portals header
50 * nid: destination node
51 * pid: destination process
52 * data: body of the message
53 * len: length of the body
54 * Returns: zero on success
56 * sends a packet to the peer, after insuring that a connection exists
58 ptl_err_t tcpnal_send(lib_nal_t *n,
71 bridge b=(bridge)n->libnal_data;
72 struct iovec tiov[257];
73 static pthread_mutex_t send_lock = PTHREAD_MUTEX_INITIALIZER;
74 ptl_err_t rc = PTL_OK;
80 if (!(c=force_tcp_connection((manager)b->lower,
86 /* TODO: these results should be checked. furthermore, provision
87 must be made for the SIGPIPE which is delivered when
88 writing on a tcp socket which has closed underneath
89 the application. there is a linux flag in the sendmsg
90 call which turns off the signally behaviour, but its
93 LASSERT (niov <= 256);
95 tiov[0].iov_base = hdr;
96 tiov[0].iov_len = sizeof(ptl_hdr_t);
97 ntiov = 1 + lib_extract_iov(256, &tiov[1], niov, iov, offset, len);
99 pthread_mutex_lock(&send_lock);
101 for (i = total = 0; i < ntiov; i++)
102 total += tiov[i].iov_len;
104 sysrc = syscall(SYS_writev, c->fd, tiov, ntiov);
105 if (sysrc != total) {
106 fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n",
111 for (i = total = 0; i <= ntiov; i++) {
112 rc = send(c->fd, tiov[i].iov_base, tiov[i].iov_len, 0);
114 if (rc != tiov[i].iov_len) {
115 fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n",
116 rc, tiov[i].iov_len, errno);
124 fprintf (stderr, "sent %s total %d in %d frags\n",
125 hdr->type == PTL_MSG_ACK ? "ACK" :
126 hdr->type == PTL_MSG_PUT ? "PUT" :
127 hdr->type == PTL_MSG_GET ? "GET" :
128 hdr->type == PTL_MSG_REPLY ? "REPLY" :
129 hdr->type == PTL_MSG_HELLO ? "HELLO" : "UNKNOWN",
132 pthread_mutex_unlock(&send_lock);
135 /* NB the NAL only calls lib_finalize() if it returns PTL_OK
137 lib_finalize(n, private, cookie, PTL_OK);
144 /* Function: tcpnal_recv
145 * Arguments: lib_nal_t *nal: pointer to my nal control block
146 * void *private: connection pointer passed through
148 * lib_msg_t *cookie: passed back to portals library
149 * user_ptr data: pointer to the destination buffer
150 * size_t mlen: length of the body
151 * size_t rlen: length of data in the network
152 * Returns: zero on success
154 * blocking read of the requested data. must drain out the
155 * difference of mainpulated and requested lengths from the network
157 ptl_err_t tcpnal_recv(lib_nal_t *n,
167 struct iovec tiov[256];
176 LASSERT(rlen >= mlen);
178 ntiov = lib_extract_iov(256, tiov, niov, iov, offset, mlen);
181 * 1. Is this effecient enough? change to use readv() directly?
182 * 2. need check return from read_connection()
185 for (i = 0; i < ntiov; i++)
186 read_connection(private, tiov[i].iov_base, tiov[i].iov_len);
189 /* FIXME; we always assume success here... */
190 lib_finalize(n, private, cookie, PTL_OK);
193 char *trash=malloc(rlen-mlen);
195 /*TODO: check error status*/
196 read_connection(private,trash,rlen-mlen);
204 /* Function: from_connection:
205 * Arguments: c: the connection to read from
206 * Returns: whether or not to continue reading from this connection,
207 * expressed as a 1 to continue, and a 0 to not
209 * from_connection() is called from the select loop when i/o is
210 * available. It attempts to read the portals header and
211 * pass it to the generic library for processing.
213 static int from_connection(void *a, void *d)
219 if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){
220 lib_parse(b->lib_nal, &hdr, c);
221 /*TODO: check error status*/
228 static void tcpnal_shutdown(bridge b)
230 shutdown_connections(b->lower);
233 /* Function: PTL_IFACE_TCP
234 * Arguments: pid_request: desired port number to bind to
235 * desired: passed NAL limits structure
236 * actual: returned NAL limits structure
237 * Returns: a nal structure on success, or null on failure
239 int tcpnal_init(bridge b)
243 b->lib_nal->libnal_send=tcpnal_send;
244 b->lib_nal->libnal_recv=tcpnal_recv;
245 b->shutdown=tcpnal_shutdown;
247 if (!(m=init_connections(PNAL_PORT(b->lib_nal->libnal_ni.ni_pid.nid,
248 b->lib_nal->libnal_ni.ni_pid.pid),
249 from_connection,b))){
250 /* TODO: this needs to shut down the
251 newly created junk */
252 return(PTL_NAL_FAILED);