1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2002 Cray Inc.
6 * This file is part of Lustre, http://www.lustre.org.
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 This file provides a simple stateful connection manager which
24 builds tcp connections on demand and leaves them open for
25 future use. It also provides the machinery to allow peers
37 #include <sys/types.h>
38 #include <sys/socket.h>
39 #include <netinet/in.h>
40 #include <netinet/tcp.h>
41 #include <portals/types.h>
42 #include <portals/lib-types.h>
43 #include <portals/socknal.h>
44 #include <libcfs/kp30.h>
45 #include <connection.h>
52 /* global variable: acceptor port */
53 unsigned short tcpnal_acceptor_port = 988;
56 /* Function: compare_connection
57 * Arguments: connection c: a connection in the hash table
58 * ptl_process_id_t: an id to verify agains
59 * Returns: 1 if the connection is the one requested, 0 otherwise
61 * compare_connection() tests for collisions in the hash table
63 static int compare_connection(void *arg1, void *arg2)
66 unsigned int * id = arg2;
68 return((c->ip==id[0]) && (c->port==id[1]));
70 /* CFS specific hacking */
71 return (c->ip == id[0]);
76 /* Function: connection_key
77 * Arguments: ptl_process_id_t id: an id to hash
78 * Returns: a not-particularily-well-distributed hash
81 static unsigned int connection_key(unsigned int *id)
86 /* CFS specific hacking */
87 return (unsigned int) id[0];
92 /* Function: remove_connection
93 * Arguments: c: the connection to remove
95 void remove_connection(void *arg)
102 hash_table_remove(c->m->connections,id);
108 /* Function: read_connection:
109 * Arguments: c: the connection to read from
110 * dest: the buffer to read into
111 * len: the number of bytes to read
112 * Returns: success as 1, or failure as 0
114 * read_connection() reads data from the connection, continuing
115 * to read partial results until the request is satisfied or
116 * it errors. TODO: this read should be covered by signal protection.
118 int read_connection(connection c,
127 rc = syscall(SYS_read, c->fd, dest+offset, len-offset);
129 rc = recv(c->fd, dest+offset, len-offset, 0);
132 if (errno == EINTR) {
135 remove_connection(c);
140 } while (offset < len);
145 static int connection_input(void *d)
148 return((*c->m->handler)(c->m->handler_arg,c));
152 /* Function: allocate_connection
153 * Arguments: t: tcpnal the allocation is occuring in the context of
154 * dest: portal endpoint address for this connection
155 * fd: open file descriptor for the socket
156 * Returns: an allocated connection structure
158 * just encompasses the action common to active and passive
159 * connections of allocation and placement in the global table
161 static connection allocate_connection(manager m,
166 connection c=malloc(sizeof(struct connection));
174 register_io_handler(fd,READ_HANDLER,connection_input,c);
175 hash_table_insert(m->connections,c,id);
180 /* Function: new_connection
181 * Arguments: t: opaque argument holding the tcpname
182 * Returns: 1 in order to reregister for new connection requests
184 * called when the bound service socket recieves
185 * a new connection request, it always accepts and
186 * installs a new connection
188 static int new_connection(void *z)
191 struct sockaddr_in s;
192 int len=sizeof(struct sockaddr_in);
193 int fd=accept(m->bound,(struct sockaddr *)&s,&len);
194 unsigned int nid=*((unsigned int *)&s.sin_addr);
195 /* cfs specific hack */
196 //unsigned short pid=s.sin_port;
197 pthread_mutex_lock(&m->conn_lock);
198 allocate_connection(m,htonl(nid),0/*pid*/,fd);
199 pthread_mutex_unlock(&m->conn_lock);
203 extern ptl_nid_t tcpnal_mynid;
206 tcpnal_hello (int sockfd, ptl_nid_t *nid, int type, __u64 incarnation)
211 ptl_magicversion_t *hmv = (ptl_magicversion_t *)&hdr.dest_nid;
213 LASSERT (sizeof (*hmv) == sizeof (hdr.dest_nid));
215 memset (&hdr, 0, sizeof (hdr));
216 hmv->magic = cpu_to_le32(PORTALS_PROTO_MAGIC);
217 hmv->version_major = cpu_to_le32(PORTALS_PROTO_VERSION_MAJOR);
218 hmv->version_minor = cpu_to_le32(PORTALS_PROTO_VERSION_MINOR);
220 hdr.src_nid = cpu_to_le64(tcpnal_mynid);
221 hdr.type = cpu_to_le32(PTL_MSG_HELLO);
223 hdr.msg.hello.type = cpu_to_le32(type);
224 hdr.msg.hello.incarnation = cpu_to_le64(incarnation);
226 /* I don't send any interface info */
228 /* Assume sufficient socket buffering for this message */
229 rc = syscall(SYS_write, sockfd, &hdr, sizeof(hdr));
231 CERROR ("Error %d sending HELLO to "LPX64"\n", rc, *nid);
235 rc = syscall(SYS_read, sockfd, hmv, sizeof(*hmv));
237 CERROR ("Error %d reading HELLO from "LPX64"\n", rc, *nid);
241 if (hmv->magic != le32_to_cpu(PORTALS_PROTO_MAGIC)) {
242 CERROR ("Bad magic %#08x (%#08x expected) from "LPX64"\n",
243 cpu_to_le32(hmv->magic), PORTALS_PROTO_MAGIC, *nid);
247 if (hmv->version_major != cpu_to_le16 (PORTALS_PROTO_VERSION_MAJOR) ||
248 hmv->version_minor != cpu_to_le16 (PORTALS_PROTO_VERSION_MINOR)) {
249 CERROR ("Incompatible protocol version %d.%d (%d.%d expected)"
251 le16_to_cpu (hmv->version_major),
252 le16_to_cpu (hmv->version_minor),
253 PORTALS_PROTO_VERSION_MAJOR,
254 PORTALS_PROTO_VERSION_MINOR,
259 #if (PORTALS_PROTO_VERSION_MAJOR != 1)
260 # error "This code only understands protocol version 1.x"
262 /* version 1 sends magic/version as the dest_nid of a 'hello' header,
263 * so read the rest of it in now... */
265 rc = syscall(SYS_read, sockfd, hmv + 1, sizeof(hdr) - sizeof(*hmv));
267 CERROR ("Error %d reading rest of HELLO hdr from "LPX64"\n",
272 /* ...and check we got what we expected */
273 if (hdr.type != cpu_to_le32 (PTL_MSG_HELLO)) {
274 CERROR ("Expecting a HELLO hdr "
275 " but got type %d with %d payload from "LPX64"\n",
276 le32_to_cpu (hdr.type),
277 le32_to_cpu (hdr.payload_length), *nid);
281 if (le64_to_cpu(hdr.src_nid) == PTL_NID_ANY) {
282 CERROR("Expecting a HELLO hdr with a NID, but got PTL_NID_ANY\n");
286 if (*nid == PTL_NID_ANY) { /* don't know peer's nid yet */
287 *nid = le64_to_cpu(hdr.src_nid);
288 } else if (*nid != le64_to_cpu (hdr.src_nid)) {
289 CERROR ("Connected to nid "LPX64", but expecting "LPX64"\n",
290 le64_to_cpu (hdr.src_nid), *nid);
294 /* Ignore any interface info in the payload */
295 nob = le32_to_cpu(hdr.payload_length);
296 if (nob > getpagesize()) {
297 CERROR("Unexpected HELLO payload %d from "LPX64"\n",
302 char *space = (char *)malloc(nob);
305 CERROR("Can't allocate scratch buffer %d\n", nob);
309 rc = syscall(SYS_read, sockfd, space, nob);
311 CERROR("Error %d skipping HELLO payload from "
312 LPX64"\n", rc, *nid);
320 /* Function: force_tcp_connection
321 * Arguments: t: tcpnal
322 * dest: portals endpoint for the connection
323 * Returns: an allocated connection structure, either
324 * a pre-existing one, or a new connection
326 connection force_tcp_connection(manager m,
332 struct sockaddr_in addr;
333 struct sockaddr_in locaddr;
342 ptl_nid_t peernid = PTL_NID_ANY;
344 port = tcpnal_acceptor_port;
349 pthread_mutex_lock(&m->conn_lock);
351 conn = hash_table_find(m->connections, id);
355 memset(&addr, 0, sizeof(addr));
356 addr.sin_family = AF_INET;
357 addr.sin_addr.s_addr = htonl(ip);
358 addr.sin_port = htons(port);
360 memset(&locaddr, 0, sizeof(locaddr));
361 locaddr.sin_family = AF_INET;
362 locaddr.sin_addr.s_addr = INADDR_ANY;
364 for (rport = IPPORT_RESERVED - 1; rport > IPPORT_RESERVED / 2; --rport) {
365 fd = socket(AF_INET, SOCK_STREAM, 0);
367 perror("tcpnal socket failed");
372 rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR,
373 &option, sizeof(option));
375 perror ("Can't set SO_REUSEADDR for socket");
380 locaddr.sin_port = htons(rport);
381 rc = bind(fd, (struct sockaddr *)&locaddr, sizeof(locaddr));
382 if (rc == 0 || errno == EACCES) {
383 rc = connect(fd, (struct sockaddr *)&addr,
384 sizeof(struct sockaddr_in));
387 } else if (errno != EADDRINUSE && errno != EADDRNOTAVAIL) {
388 perror("Error connecting to remote host");
392 } else if (errno != EADDRINUSE) {
393 perror("Error binding to privileged port");
400 if (rport == IPPORT_RESERVED / 2) {
401 fprintf(stderr, "Out of ports trying to bind to a reserved port\n");
407 setsockopt(fd, SOL_TCP, TCP_NODELAY, &option, sizeof(option));
409 setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &option, sizeof(option));
411 setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &option, sizeof(option));
414 gettimeofday(&tv, NULL);
415 incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
418 if (tcpnal_hello(fd, &peernid, SOCKNAL_CONN_ANY, incarnation))
421 conn = allocate_connection(m, ip, port, fd);
423 /* let nal thread know this event right away */
425 procbridge_wakeup_nal(pb);
428 pthread_mutex_unlock(&m->conn_lock);
433 /* Function: bind_socket
434 * Arguments: t: the nal state for this interface
435 * port: the port to attempt to bind to
436 * Returns: 1 on success, or 0 on error
438 * bind_socket() attempts to allocate and bind a socket to the requested
439 * port, or dynamically assign one from the kernel should the port be
440 * zero. Sets the bound and bound_handler elements of m.
442 * TODO: The port should be an explicitly sized type.
444 static int bind_socket(manager m,unsigned short port)
446 struct sockaddr_in addr;
447 int alen=sizeof(struct sockaddr_in);
449 if ((m->bound = socket(AF_INET, SOCK_STREAM, 0)) < 0)
452 bzero((char *) &addr, sizeof(addr));
453 addr.sin_family = AF_INET;
454 addr.sin_addr.s_addr = 0;
455 addr.sin_port = htons(port);
457 if (bind(m->bound,(struct sockaddr *)&addr,alen)<0){
458 perror ("tcpnal bind");
462 getsockname(m->bound,(struct sockaddr *)&addr, &alen);
464 m->bound_handler=register_io_handler(m->bound,READ_HANDLER,
467 m->port=addr.sin_port;
472 /* Function: shutdown_connections
473 * Arguments: m: the manager structure
475 * close all connections and reclaim resources
477 void shutdown_connections(manager m)
480 remove_io_handler(m->bound_handler);
481 hash_destroy_table(m->connections,remove_connection);
486 /* Function: init_connections
487 * Arguments: t: the nal state for this interface
488 * port: the port to attempt to bind to
489 * Returns: a newly allocated manager structure, or
490 * zero if the fixed port could not be bound
492 manager init_connections(unsigned short pid,
493 int (*input)(void *, void *),
496 manager m = (manager)malloc(sizeof(struct manager));
497 m->connections = hash_create_table(compare_connection,connection_key);
500 pthread_mutex_init(&m->conn_lock, 0);
502 if (bind_socket(m,pid))