1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2002 Cray Inc.
5 * Copyright (c) 2003 Cluster File Systems, Inc.
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 This file implements the TCP-based nal by providing glue
25 between the connection service and the generic NAL implementation */
31 #include <sys/types.h>
32 #include <sys/socket.h>
33 #include <netinet/in.h>
36 #include <procbridge.h>
37 #include <connection.h>
45 tcpnal_notify(lnet_ni_t *ni, lnet_nid_t nid, int alive)
47 bridge b = (bridge)ni->ni_data;
54 c = force_tcp_connection((manager)b->lower, nid, b->local);
56 CERROR("Can't create connection to %s\n",
61 * sends a packet to the peer, after insuring that a connection exists
63 int tcpnal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
65 lnet_hdr_t *hdr = &lntmsg->msg_hdr;
66 lnet_process_id_t target = lntmsg->msg_target;
67 unsigned int niov = lntmsg->msg_niov;
68 struct iovec *iov = lntmsg->msg_iov;
69 unsigned int offset = lntmsg->msg_offset;
70 unsigned int len = lntmsg->msg_len;
73 bridge b = (bridge)ni->ni_data;
74 struct iovec tiov[257];
75 static pthread_mutex_t send_lock = PTHREAD_MUTEX_INITIALIZER;
82 if (!(c = force_tcp_connection((manager)b->lower, target.nid,
86 /* TODO: these results should be checked. furthermore, provision
87 must be made for the SIGPIPE which is delivered when
88 writing on a tcp socket which has closed underneath
89 the application. there is a linux flag in the sendmsg
90 call which turns off the signally behaviour, but its
93 LASSERT (niov <= 256);
94 LASSERT (len == 0 || iov != NULL); /* I don't understand kiovs */
96 tiov[0].iov_base = hdr;
97 tiov[0].iov_len = sizeof(lnet_hdr_t);
98 ntiov = 1 + lnet_extract_iov(256, &tiov[1], niov, iov, offset, len);
100 pthread_mutex_lock(&send_lock);
102 for (i = total = 0; i < ntiov; i++)
103 total += tiov[i].iov_len;
105 sysrc = syscall(SYS_writev, c->fd, tiov, ntiov);
106 if (sysrc != total) {
107 fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n",
112 for (i = total = 0; i <= ntiov; i++) {
113 rc = send(c->fd, tiov[i].iov_base, tiov[i].iov_len, 0);
115 if (rc != tiov[i].iov_len) {
116 fprintf (stderr, "BAD SEND rc %d != %d, errno %d\n",
117 rc, tiov[i].iov_len, errno);
125 fprintf (stderr, "sent %s total %d in %d frags\n",
126 hdr->type == LNET_MSG_ACK ? "ACK" :
127 hdr->type == LNET_MSG_PUT ? "PUT" :
128 hdr->type == LNET_MSG_GET ? "GET" :
129 hdr->type == LNET_MSG_REPLY ? "REPLY" :
130 hdr->type == LNET_MSG_HELLO ? "HELLO" : "UNKNOWN",
133 pthread_mutex_unlock(&send_lock);
136 /* NB the NAL only calls lnet_finalize() if it returns 0
138 lnet_finalize(ni, lntmsg, 0);
145 int tcpnal_recv(lnet_ni_t *ni,
156 struct iovec tiov[256];
163 LASSERT(iov != NULL); /* I don't understand kiovs */
165 ntiov = lnet_extract_iov(256, tiov, niov, iov, offset, mlen);
168 * 1. Is this effecient enough? change to use readv() directly?
171 for (i = 0; i < ntiov; i++)
172 if (!read_connection(private, tiov[i].iov_base, tiov[i].iov_len))
177 LASSERT(rlen >= mlen);
181 char *trash=malloc(rlen - mlen);
186 rc = read_connection(private, trash, rlen - mlen);
192 lnet_finalize(ni, cookie, 0);
197 /* Function: from_connection:
198 * Arguments: c: the connection to read from
199 * Returns: whether or not to continue reading from this connection,
200 * expressed as a 1 to continue, and a 0 to not
202 * from_connection() is called from the select loop when i/o is
203 * available. It attempts to read the portals header and
204 * pass it to the generic library for processing.
206 static int from_connection(void *a, void *d)
213 if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))) {
214 /* replace dest_nid,pid (socknal sets its own) */
215 hdr.dest_nid = cpu_to_le64(b->b_ni->ni_nid);
216 hdr.dest_pid = cpu_to_le32(the_lnet.ln_pid);
218 rc = lnet_parse(b->b_ni, &hdr, c->peer_nid, c, 0);
220 CERROR("Error %d from lnet_parse\n", rc);
230 void tcpnal_shutdown(bridge b)
232 shutdown_connections(b->lower);
235 /* Function: PTL_IFACE_TCP
236 * Arguments: pid_request: desired port number to bind to
237 * desired: passed NAL limits structure
238 * actual: returned NAL limits structure
239 * Returns: a nal structure on success, or null on failure
241 int tcpnal_init(bridge b)
245 tcpnal_set_global_params();
247 if (!(m = init_connections(from_connection, b))) {
248 /* TODO: this needs to shut down the newly created junk */