From 271d764b97d4ba76e9fca73a03217eccbfcdb5f9 Mon Sep 17 00:00:00 2001 From: ericm Date: Fri, 22 Aug 2003 15:02:33 +0000 Subject: [PATCH] merge b_devel -> b_eq: 20030822 --- lnet/klnds/scimaclnd/README.scimacnal | 34 +++- lnet/klnds/scimaclnd/scimacnal.c | 30 +++- lnet/klnds/scimaclnd/scimacnal.h | 9 +- lnet/klnds/scimaclnd/scimacnal_cb.c | 215 +++++++++++++++++++----- lnet/libcfs/debug.c | 16 +- lnet/libcfs/module.c | 16 +- lnet/libcfs/proc.c | 16 +- lnet/ulnds/address.c | 16 +- lnet/ulnds/connection.c | 16 +- lnet/ulnds/debug.c | 16 +- lnet/ulnds/pqtimer.c | 16 +- lnet/ulnds/procapi.c | 16 +- lnet/ulnds/proclib.c | 16 +- lnet/ulnds/select.c | 16 +- lnet/ulnds/socklnd/address.c | 16 +- lnet/ulnds/socklnd/connection.c | 16 +- lnet/ulnds/socklnd/debug.c | 16 +- lnet/ulnds/socklnd/pqtimer.c | 16 +- lnet/ulnds/socklnd/procapi.c | 16 +- lnet/ulnds/socklnd/proclib.c | 16 +- lnet/ulnds/socklnd/select.c | 16 +- lnet/ulnds/socklnd/table.c | 16 +- lnet/ulnds/socklnd/tcplnd.c | 16 +- lnet/ulnds/table.c | 16 +- lnet/ulnds/tcplnd.c | 16 +- lustre/portals/knals/scimacnal/README.scimacnal | 34 +++- lustre/portals/knals/scimacnal/scimacnal.c | 30 +++- lustre/portals/knals/scimacnal/scimacnal.h | 9 +- lustre/portals/knals/scimacnal/scimacnal_cb.c | 215 +++++++++++++++++++----- lustre/portals/libcfs/debug.c | 16 +- lustre/portals/libcfs/module.c | 16 +- lustre/portals/libcfs/proc.c | 16 +- lustre/portals/unals/address.c | 16 +- lustre/portals/unals/connection.c | 16 +- lustre/portals/unals/debug.c | 16 +- lustre/portals/unals/pqtimer.c | 16 +- lustre/portals/unals/procapi.c | 16 +- lustre/portals/unals/proclib.c | 16 +- lustre/portals/unals/select.c | 16 +- lustre/portals/unals/table.c | 16 +- lustre/portals/unals/tcpnal.c | 16 +- lustre/ptlrpc/import.c | 204 ++++++++++++++++++++++ 42 files changed, 944 insertions(+), 364 deletions(-) create mode 100644 lustre/ptlrpc/import.c diff --git a/lnet/klnds/scimaclnd/README.scimacnal b/lnet/klnds/scimaclnd/README.scimacnal index d4c6a49..e1ee3b5 100644 --- a/lnet/klnds/scimaclnd/README.scimacnal +++ b/lnet/klnds/scimaclnd/README.scimacnal @@ -2,13 +2,41 @@ scimacnal - A NAL for the Scali ScaMAC midlayer. The ScaMAC midlayer is a simplified API to the SCI high performance -interconnect. +interconnect (http://www.scali.com/, http://www.dolphinics.com/). In order to use this NAL you'll need to tune scimac to use larger buffers. See scimac.conf in this directory for an example. -Overall performance and stability isn't great but this can be attributed -to the scimac driver which apparently is in need of some development. +You'll also need to edit portals/include/portals/lib-types.h and reduce +the MTU to 64kB (the limit of scimac), this diff should help: + +----------------------8<---------------------------------- +--- portals/include/portals/lib-types.h 27 Jul 2003 02:05:47 -0000 1.1.2.5 ++++ portals/include/portals/lib-types.h 14 Aug 2003 08:32:14 -0000 +@@ -137,8 +137,8 @@ + } lib_counters_t; + + /* temporary expedient: limit number of entries in discontiguous MDs */ +-# define PTL_MTU (512<<10) +-# define PTL_MD_MAX_IOV 128 ++# define PTL_MTU (64<<10) ++# define PTL_MD_MAX_IOV 16 + + struct lib_msg_t { + struct list_head msg_list; +----------------------8<---------------------------------- + +The NAL itself seems quite stable, though scimac has recovery bugs when +rebooting nodes at times (confirmed by the fact that the IP driver that +also uses scimac loses connectivity when scimacnal does). This is +solved by unloading lustre and reloading the scimac driver on the +affected nodes. + +Performance isn't great when it comes to latency, scimac seems to have +problems with per packet latencies (confirmed with the IP driver which +has similar behaviour). Bandwidth using large packets is pretty OK +(probably due to the fact that it hides the latency issues). TODO: Routing isn't yet implemented. +Need some way to inform portals about our MTU. diff --git a/lnet/klnds/scimaclnd/scimacnal.c b/lnet/klnds/scimaclnd/scimacnal.c index 479cc2c..f3fe617 100644 --- a/lnet/klnds/scimaclnd/scimacnal.c +++ b/lnet/klnds/scimaclnd/scimacnal.c @@ -38,6 +38,24 @@ kpr_nal_interface_t kscimacnal_router_interface = { }; +int kscimacnal_cmd (struct portal_ioctl_data *data, void *private) +{ + LASSERT (data != NULL); + + switch (data->ioc_nal_cmd) { + case NAL_CMD_REGISTER_MYNID: + if(kscimacnal_lib.ni.nid == data->ioc_nid) { + break; + } + CDEBUG (D_IOCTL, "Can't change NID from "LPX64" to "LPX64")\n", kscimacnal_lib.ni.nid, data->ioc_nid); + return(-EINVAL); + default: + return(-EINVAL); + } + + return(0); +} + static int kscimacnal_forward(nal_t *nal, int id, void *args, size_t args_len, @@ -200,6 +218,16 @@ kscimacnal_initialize(void) return (-ENOMEM); } + /* Init command interface */ + rc = kportal_nal_register (SCIMACNAL, &kscimacnal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + PtlNIFini(kscimacnal_ni); + mac_finish(machandle); + return (rc); + } + + PORTAL_SYMBOL_REGISTER(kscimacnal_ni); /* We're done now, it's OK for the RX callback to do stuff */ @@ -210,7 +238,7 @@ kscimacnal_initialize(void) MODULE_AUTHOR("Niklas Edmundsson "); -MODULE_DESCRIPTION("Kernel Scali ScaMAC SCI NAL v0.0"); +MODULE_DESCRIPTION("Kernel Scali ScaMAC SCI NAL v0.1"); MODULE_LICENSE("GPL"); module_init (kscimacnal_initialize); diff --git a/lnet/klnds/scimaclnd/scimacnal.h b/lnet/klnds/scimaclnd/scimacnal.h index 1ff180e..6949557 100644 --- a/lnet/klnds/scimaclnd/scimacnal.h +++ b/lnet/klnds/scimaclnd/scimacnal.h @@ -43,10 +43,10 @@ #define MAC_SAPID_LUSTRE MAC_SAPID_TEST1 #endif /* MAC_SAPID_LUSTRE */ +/* scimac has an annoying MTU limit of 64k */ #define SCIMACNAL_MTU 65536 -/* FIXME: What is really the MTU of lustre? */ -#if PTL_MD_MAX_IOV*PAGE_SIZE > SCIMACNAL_MTU -#error Max MTU of ScaMAC is 64k, PTL_MD_MAX_IOV*PAGE_SIZE is bigger. +#if PTL_MTU > SCIMACNAL_MTU +#error Max MTU of ScaMAC is 64k, PTL_MTU is bigger. #endif typedef struct { @@ -62,6 +62,9 @@ typedef struct { void *ktx_private; lib_msg_t *ktx_cookie; ptl_hdr_t ktx_hdr; + /* To be able to kunmap() kmap():ed pages */ + struct page *ktx_kpages[PTL_MD_MAX_IOV]; + int ktx_nmapped; } kscimacnal_tx_t; diff --git a/lnet/klnds/scimaclnd/scimacnal_cb.c b/lnet/klnds/scimaclnd/scimacnal_cb.c index 7e4a2e8..cc0c102 100644 --- a/lnet/klnds/scimaclnd/scimacnal_cb.c +++ b/lnet/klnds/scimaclnd/scimacnal_cb.c @@ -156,9 +156,15 @@ static void kscimacnal_txrelease(mac_mblk_t *msg, mac_msg_status_t status, void *context) { kscimacnal_tx_t *ktx = (kscimacnal_tx_t *)context; - int err=0; + int err=0, i; LASSERT (ktx != NULL); + /* Unmap any mapped pages */ + for(i=0; iktx_nmapped; i++) { + kunmap(ktx->ktx_kpages[i]); + } + + CDEBUG(D_NET, "kunmapped %d pages\n", ktx->ktx_nmapped); /* Euh, there is no feedback when transmission fails?! */ switch(status) { @@ -178,17 +184,21 @@ kscimacnal_txrelease(mac_mblk_t *msg, mac_msg_status_t status, void *context) /* Called by portals when it wants to send a message. * Since ScaMAC has it's own TX thread we don't bother setting up our own. */ -static int -kscimacnal_send(nal_cb_t *nal, - void *private, - lib_msg_t *cookie, - ptl_hdr_t *hdr, - int type, - ptl_nid_t nid, - ptl_pid_t pid, - unsigned int payload_niov, - struct iovec *payload_iov, - size_t payload_len) + +/* FIXME: Read comments in qswnal_cb.c for _sendmsg and fix return-on-error + * issues */ +static inline int +kscimacnal_sendmsg(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + ptl_kiov_t *payload_kiov, + size_t payload_len) { kscimacnal_tx_t *ktx=NULL; kscimacnal_data_t *ksci = nal->nal_data; @@ -198,12 +208,18 @@ kscimacnal_send(nal_cb_t *nal, unsigned long physaddr; - CDEBUG(D_NET, "sending %d bytes from %p to nid 0x%Lx niov: %d\n", - payload_len, payload_iov, nid, payload_niov); + CDEBUG(D_NET, "sending %d bytes from %p/%p to nid 0x%Lx niov: %d\n", + payload_len, payload_iov, payload_kiov, nid, payload_niov); + /* Basic sanity checks */ LASSERT(ksci != NULL); - LASSERT(hdr != NULL); + LASSERT (payload_len == 0 || payload_niov > 0); + LASSERT (payload_niov <= PTL_MD_MAX_IOV); + /* It must be OK to kmap() if required */ + LASSERT (payload_kiov == NULL || !in_interrupt ()); + /* payload is either all vaddrs or all pages */ + LASSERT (!(payload_kiov != NULL && payload_iov != NULL)); /* Do real check if we can send this */ if (buf_len > mac_get_mtusize(ksci->ksci_machandle)) { @@ -219,6 +235,8 @@ kscimacnal_send(nal_cb_t *nal, return -ENOMEM; } + ktx->ktx_nmapped = 0; /* Start with no mapped pages :) */ + /* *SIGH* hdr is a stack variable in the calling function, so we * need to copy it to a buffer. Zerocopy magic (or is it just * deferred memcpy?) is annoying sometimes. */ @@ -235,19 +253,34 @@ kscimacnal_send(nal_cb_t *nal, lastblk=msg; /* Allocate additional mblks for each iov as needed. - * Essentially lib_copy_iov2buf with a twist or two */ + * Essentially lib_copy_(k)iov2buf with a twist or two */ while (payload_len > 0) { - ptl_size_t nob; + ptl_size_t nob; + char *addr; LASSERT (payload_niov > 0); - nob = MIN (payload_iov->iov_len, payload_len); + if(payload_iov != NULL) { + nob = MIN (payload_iov->iov_len, payload_len); + addr = payload_iov->iov_base; + } + else { + nob = MIN (payload_kiov->kiov_len, payload_len); + /* Bollocks. We need to handle paged IO for things to + * work but there is no good way to do this. We + * do it by kmap():ing all pages and keep them + * mapped until scimac is done with them. */ + /* FIXME: kunmap() on error */ + addr = kmap(payload_kiov->kiov_page); + ktx->ktx_kpages[ktx->ktx_nmapped++] = + payload_kiov->kiov_page; + } + /* We don't need a callback on the additional mblks, + * since all release callbacks seems to be called when + * the entire message has been sent */ + newblk=mac_alloc_mblk(addr, nob, NULL, NULL); - /* We don't need a callback on the additional mblks, since - * all release callbacks seems to be called when the entire - * message has been sent */ - newblk=mac_alloc_mblk(payload_iov->iov_base, nob, NULL, NULL); if(!newblk) { mac_free_msg(msg); PORTAL_FREE(ktx, (sizeof(kscimacnal_tx_t))); @@ -259,9 +292,16 @@ kscimacnal_send(nal_cb_t *nal, payload_len -= nob; payload_niov--; - payload_iov++; + if(payload_iov != NULL) { + payload_iov++; + } + else { + payload_kiov++; + } } + CDEBUG(D_NET, "kmapped %d pages\n", ktx->ktx_nmapped); + ktx->ktx_nal = nal; ktx->ktx_private = private; ktx->ktx_cookie = cookie; @@ -282,6 +322,39 @@ kscimacnal_send(nal_cb_t *nal, } +static int +kscimacnal_send (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_nob) +{ + return (kscimacnal_sendmsg (nal, private, cookie, hdr, type, nid, pid, + payload_niov, payload_iov, NULL, payload_nob)); +} + +static int +kscimacnal_send_pages (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_nob) +{ + return (kscimacnal_sendmsg (nal, private, cookie, hdr, type, nid, pid, + payload_niov, NULL, payload_kiov, payload_nob)); +} + + void kscimacnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd) { @@ -366,19 +439,22 @@ kscimacnal_rx(mac_handle_t *handle, mac_mblk_t *msg, mac_msg_type_t type, /* Called by portals to process a recieved packet */ -static int kscimacnal_recv(nal_cb_t *nal, - void *private, - lib_msg_t *cookie, - unsigned int niov, - struct iovec *iov, - size_t mlen, - size_t rlen) +inline static int +kscimacnal_recvmsg(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + struct iovec *iov, + ptl_kiov_t *kiov, + size_t mlen, + size_t rlen) { kscimacnal_rx_t *krx = private; mac_mblk_t *mblk; void *src; mac_size_t pkt_len; ptl_size_t iovused=0; + char *base=NULL; LASSERT (krx != NULL); LASSERT (krx->msg != NULL); @@ -393,6 +469,10 @@ static int kscimacnal_recv(nal_cb_t *nal, */ LASSERT (mlen==0 || mac_msg_size(krx->msg) >= sizeof(ptl_hdr_t)+rlen); LASSERT (mlen==0 || mlen <= rlen); + /* It must be OK to kmap() if required */ + LASSERT (kiov == NULL || !in_interrupt ()); + /* Either all pages or all vaddrs */ + LASSERT (!(kiov != NULL && iov != NULL)); PROF_START(memcpy); @@ -407,36 +487,59 @@ static int kscimacnal_recv(nal_cb_t *nal, LASSERT(src != NULL); - /* Essentially lib_copy_buf2iov but with continuation support, - * we "gracefully" thrash the argument vars ;) */ + /* Essentially lib_copy_buf2(k)iov but with continuation + * support, we "gracefully" thrash the argument vars ;) */ while (pkt_len > 0) { - ptl_size_t nob; + ptl_size_t nob, len; LASSERT (niov > 0); - LASSERT(iovused < iov->iov_len); + if(iov != NULL) { + LASSERT(iovused < iov->iov_len); + len = iov->iov_len; + base = iov->iov_base; + } + else { + LASSERT(iovused < kiov->kiov_len); + len = kiov->kiov_len; + if(base==NULL) { + /* New page */ + base = kmap(kiov->kiov_page); + } + } - nob = MIN (iov->iov_len-iovused, pkt_len); - CDEBUG(D_NET, "iovbase: %p iovlen: %d src: %p nob: %d " + nob = MIN (len-iovused, pkt_len); + CDEBUG(D_NET, "base: %p len: %d src: %p nob: %d " "iovused: %d\n", - iov->iov_base, iov->iov_len, - src, nob, iovused); + base, len, src, nob, iovused); - memcpy (iov->iov_base+iovused, src, nob); + memcpy (base+iovused, src, nob); pkt_len -= nob; src += nob; - if(nob+iovused < iov->iov_len) { + if(nob+iovused < len) { /* We didn't use all of the iov */ iovused+=nob; } else { niov--; - iov++; iovused=0; + if(iov != NULL) { + iov++; + } + else { + kunmap(kiov->kiov_page); + base=NULL; + kiov++; + } } } } + /* Just to make sure the last page is unmapped */ + if(kiov!=NULL && base!=NULL) { + kunmap(kiov->kiov_page); + base=NULL; + } PROF_FINISH(memcpy); CDEBUG(D_NET, "Calling lib_finalize.\n"); @@ -451,12 +554,38 @@ static int kscimacnal_recv(nal_cb_t *nal, } +static int +kscimacnal_recv(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + struct iovec *iov, + size_t mlen, + size_t rlen) +{ + return (kscimacnal_recvmsg (nal, private, cookie, niov, iov, NULL, mlen, rlen)); +} + + +static int +kscimacnal_recv_pages (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + ptl_kiov_t *kiov, + size_t mlen, + size_t rlen) +{ + return (kscimacnal_recvmsg (nal, private, cookie, niov, NULL, kiov, mlen, rlen)); +} + + nal_cb_t kscimacnal_lib = { nal_data: &kscimacnal_data, /* NAL private data */ cb_send: kscimacnal_send, - cb_send_pages: NULL, /* Ignore for now */ + cb_send_pages: kscimacnal_send_pages, cb_recv: kscimacnal_recv, - cb_recv_pages: NULL, + cb_recv_pages: kscimacnal_recv_pages, cb_read: kscimacnal_read, cb_write: kscimacnal_write, cb_malloc: kscimacnal_malloc, diff --git a/lnet/libcfs/debug.c b/lnet/libcfs/debug.c index 9d15c8f..e9b0e12 100644 --- a/lnet/libcfs/debug.c +++ b/lnet/libcfs/debug.c @@ -4,19 +4,19 @@ * Copyright (C) 2002 Cluster File Systems, Inc. * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/libcfs/module.c b/lnet/libcfs/module.c index 9bb7e00..7ecbe43 100644 --- a/lnet/libcfs/module.c +++ b/lnet/libcfs/module.c @@ -3,19 +3,19 @@ * * Copyright (C) 2001, 2002 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/libcfs/proc.c b/lnet/libcfs/proc.c index bb6c377..8817ace 100644 --- a/lnet/libcfs/proc.c +++ b/lnet/libcfs/proc.c @@ -6,19 +6,19 @@ * Author: Peter J. Braam * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/address.c b/lnet/ulnds/address.c index b422c3f..6507924 100644 --- a/lnet/ulnds/address.c +++ b/lnet/ulnds/address.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/connection.c b/lnet/ulnds/connection.c index 1d589bf..efc8a54 100644 --- a/lnet/ulnds/connection.c +++ b/lnet/ulnds/connection.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/debug.c b/lnet/ulnds/debug.c index b73f042..b82bb2f 100644 --- a/lnet/ulnds/debug.c +++ b/lnet/ulnds/debug.c @@ -4,19 +4,19 @@ * Copyright (C) 2002 Cluster File Systems, Inc. * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/pqtimer.c b/lnet/ulnds/pqtimer.c index fa2fb4f..98c48eb 100644 --- a/lnet/ulnds/pqtimer.c +++ b/lnet/ulnds/pqtimer.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c index 4278822..206aea0 100644 --- a/lnet/ulnds/procapi.c +++ b/lnet/ulnds/procapi.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/proclib.c b/lnet/ulnds/proclib.c index dcd64ed..2424145 100644 --- a/lnet/ulnds/proclib.c +++ b/lnet/ulnds/proclib.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/select.c b/lnet/ulnds/select.c index 1dd547a..fe24efc 100644 --- a/lnet/ulnds/select.c +++ b/lnet/ulnds/select.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/address.c b/lnet/ulnds/socklnd/address.c index b422c3f..6507924 100644 --- a/lnet/ulnds/socklnd/address.c +++ b/lnet/ulnds/socklnd/address.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/connection.c b/lnet/ulnds/socklnd/connection.c index 1d589bf..efc8a54 100644 --- a/lnet/ulnds/socklnd/connection.c +++ b/lnet/ulnds/socklnd/connection.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/debug.c b/lnet/ulnds/socklnd/debug.c index b73f042..b82bb2f 100644 --- a/lnet/ulnds/socklnd/debug.c +++ b/lnet/ulnds/socklnd/debug.c @@ -4,19 +4,19 @@ * Copyright (C) 2002 Cluster File Systems, Inc. * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/pqtimer.c b/lnet/ulnds/socklnd/pqtimer.c index fa2fb4f..98c48eb 100644 --- a/lnet/ulnds/socklnd/pqtimer.c +++ b/lnet/ulnds/socklnd/pqtimer.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c index 4278822..206aea0 100644 --- a/lnet/ulnds/socklnd/procapi.c +++ b/lnet/ulnds/socklnd/procapi.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/proclib.c b/lnet/ulnds/socklnd/proclib.c index dcd64ed..2424145 100644 --- a/lnet/ulnds/socklnd/proclib.c +++ b/lnet/ulnds/socklnd/proclib.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/select.c b/lnet/ulnds/socklnd/select.c index 1dd547a..fe24efc 100644 --- a/lnet/ulnds/socklnd/select.c +++ b/lnet/ulnds/socklnd/select.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/table.c b/lnet/ulnds/socklnd/table.c index bef13c5..662775a 100644 --- a/lnet/ulnds/socklnd/table.c +++ b/lnet/ulnds/socklnd/table.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/socklnd/tcplnd.c b/lnet/ulnds/socklnd/tcplnd.c index f14a1fc..ec67a21 100644 --- a/lnet/ulnds/socklnd/tcplnd.c +++ b/lnet/ulnds/socklnd/tcplnd.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/table.c b/lnet/ulnds/table.c index bef13c5..662775a 100644 --- a/lnet/ulnds/table.c +++ b/lnet/ulnds/table.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lnet/ulnds/tcplnd.c b/lnet/ulnds/tcplnd.c index f14a1fc..ec67a21 100644 --- a/lnet/ulnds/tcplnd.c +++ b/lnet/ulnds/tcplnd.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/knals/scimacnal/README.scimacnal b/lustre/portals/knals/scimacnal/README.scimacnal index d4c6a49..e1ee3b5 100644 --- a/lustre/portals/knals/scimacnal/README.scimacnal +++ b/lustre/portals/knals/scimacnal/README.scimacnal @@ -2,13 +2,41 @@ scimacnal - A NAL for the Scali ScaMAC midlayer. The ScaMAC midlayer is a simplified API to the SCI high performance -interconnect. +interconnect (http://www.scali.com/, http://www.dolphinics.com/). In order to use this NAL you'll need to tune scimac to use larger buffers. See scimac.conf in this directory for an example. -Overall performance and stability isn't great but this can be attributed -to the scimac driver which apparently is in need of some development. +You'll also need to edit portals/include/portals/lib-types.h and reduce +the MTU to 64kB (the limit of scimac), this diff should help: + +----------------------8<---------------------------------- +--- portals/include/portals/lib-types.h 27 Jul 2003 02:05:47 -0000 1.1.2.5 ++++ portals/include/portals/lib-types.h 14 Aug 2003 08:32:14 -0000 +@@ -137,8 +137,8 @@ + } lib_counters_t; + + /* temporary expedient: limit number of entries in discontiguous MDs */ +-# define PTL_MTU (512<<10) +-# define PTL_MD_MAX_IOV 128 ++# define PTL_MTU (64<<10) ++# define PTL_MD_MAX_IOV 16 + + struct lib_msg_t { + struct list_head msg_list; +----------------------8<---------------------------------- + +The NAL itself seems quite stable, though scimac has recovery bugs when +rebooting nodes at times (confirmed by the fact that the IP driver that +also uses scimac loses connectivity when scimacnal does). This is +solved by unloading lustre and reloading the scimac driver on the +affected nodes. + +Performance isn't great when it comes to latency, scimac seems to have +problems with per packet latencies (confirmed with the IP driver which +has similar behaviour). Bandwidth using large packets is pretty OK +(probably due to the fact that it hides the latency issues). TODO: Routing isn't yet implemented. +Need some way to inform portals about our MTU. diff --git a/lustre/portals/knals/scimacnal/scimacnal.c b/lustre/portals/knals/scimacnal/scimacnal.c index 479cc2c..f3fe617 100644 --- a/lustre/portals/knals/scimacnal/scimacnal.c +++ b/lustre/portals/knals/scimacnal/scimacnal.c @@ -38,6 +38,24 @@ kpr_nal_interface_t kscimacnal_router_interface = { }; +int kscimacnal_cmd (struct portal_ioctl_data *data, void *private) +{ + LASSERT (data != NULL); + + switch (data->ioc_nal_cmd) { + case NAL_CMD_REGISTER_MYNID: + if(kscimacnal_lib.ni.nid == data->ioc_nid) { + break; + } + CDEBUG (D_IOCTL, "Can't change NID from "LPX64" to "LPX64")\n", kscimacnal_lib.ni.nid, data->ioc_nid); + return(-EINVAL); + default: + return(-EINVAL); + } + + return(0); +} + static int kscimacnal_forward(nal_t *nal, int id, void *args, size_t args_len, @@ -200,6 +218,16 @@ kscimacnal_initialize(void) return (-ENOMEM); } + /* Init command interface */ + rc = kportal_nal_register (SCIMACNAL, &kscimacnal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + PtlNIFini(kscimacnal_ni); + mac_finish(machandle); + return (rc); + } + + PORTAL_SYMBOL_REGISTER(kscimacnal_ni); /* We're done now, it's OK for the RX callback to do stuff */ @@ -210,7 +238,7 @@ kscimacnal_initialize(void) MODULE_AUTHOR("Niklas Edmundsson "); -MODULE_DESCRIPTION("Kernel Scali ScaMAC SCI NAL v0.0"); +MODULE_DESCRIPTION("Kernel Scali ScaMAC SCI NAL v0.1"); MODULE_LICENSE("GPL"); module_init (kscimacnal_initialize); diff --git a/lustre/portals/knals/scimacnal/scimacnal.h b/lustre/portals/knals/scimacnal/scimacnal.h index 1ff180e..6949557 100644 --- a/lustre/portals/knals/scimacnal/scimacnal.h +++ b/lustre/portals/knals/scimacnal/scimacnal.h @@ -43,10 +43,10 @@ #define MAC_SAPID_LUSTRE MAC_SAPID_TEST1 #endif /* MAC_SAPID_LUSTRE */ +/* scimac has an annoying MTU limit of 64k */ #define SCIMACNAL_MTU 65536 -/* FIXME: What is really the MTU of lustre? */ -#if PTL_MD_MAX_IOV*PAGE_SIZE > SCIMACNAL_MTU -#error Max MTU of ScaMAC is 64k, PTL_MD_MAX_IOV*PAGE_SIZE is bigger. +#if PTL_MTU > SCIMACNAL_MTU +#error Max MTU of ScaMAC is 64k, PTL_MTU is bigger. #endif typedef struct { @@ -62,6 +62,9 @@ typedef struct { void *ktx_private; lib_msg_t *ktx_cookie; ptl_hdr_t ktx_hdr; + /* To be able to kunmap() kmap():ed pages */ + struct page *ktx_kpages[PTL_MD_MAX_IOV]; + int ktx_nmapped; } kscimacnal_tx_t; diff --git a/lustre/portals/knals/scimacnal/scimacnal_cb.c b/lustre/portals/knals/scimacnal/scimacnal_cb.c index 7e4a2e8..cc0c102 100644 --- a/lustre/portals/knals/scimacnal/scimacnal_cb.c +++ b/lustre/portals/knals/scimacnal/scimacnal_cb.c @@ -156,9 +156,15 @@ static void kscimacnal_txrelease(mac_mblk_t *msg, mac_msg_status_t status, void *context) { kscimacnal_tx_t *ktx = (kscimacnal_tx_t *)context; - int err=0; + int err=0, i; LASSERT (ktx != NULL); + /* Unmap any mapped pages */ + for(i=0; iktx_nmapped; i++) { + kunmap(ktx->ktx_kpages[i]); + } + + CDEBUG(D_NET, "kunmapped %d pages\n", ktx->ktx_nmapped); /* Euh, there is no feedback when transmission fails?! */ switch(status) { @@ -178,17 +184,21 @@ kscimacnal_txrelease(mac_mblk_t *msg, mac_msg_status_t status, void *context) /* Called by portals when it wants to send a message. * Since ScaMAC has it's own TX thread we don't bother setting up our own. */ -static int -kscimacnal_send(nal_cb_t *nal, - void *private, - lib_msg_t *cookie, - ptl_hdr_t *hdr, - int type, - ptl_nid_t nid, - ptl_pid_t pid, - unsigned int payload_niov, - struct iovec *payload_iov, - size_t payload_len) + +/* FIXME: Read comments in qswnal_cb.c for _sendmsg and fix return-on-error + * issues */ +static inline int +kscimacnal_sendmsg(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + ptl_kiov_t *payload_kiov, + size_t payload_len) { kscimacnal_tx_t *ktx=NULL; kscimacnal_data_t *ksci = nal->nal_data; @@ -198,12 +208,18 @@ kscimacnal_send(nal_cb_t *nal, unsigned long physaddr; - CDEBUG(D_NET, "sending %d bytes from %p to nid 0x%Lx niov: %d\n", - payload_len, payload_iov, nid, payload_niov); + CDEBUG(D_NET, "sending %d bytes from %p/%p to nid 0x%Lx niov: %d\n", + payload_len, payload_iov, payload_kiov, nid, payload_niov); + /* Basic sanity checks */ LASSERT(ksci != NULL); - LASSERT(hdr != NULL); + LASSERT (payload_len == 0 || payload_niov > 0); + LASSERT (payload_niov <= PTL_MD_MAX_IOV); + /* It must be OK to kmap() if required */ + LASSERT (payload_kiov == NULL || !in_interrupt ()); + /* payload is either all vaddrs or all pages */ + LASSERT (!(payload_kiov != NULL && payload_iov != NULL)); /* Do real check if we can send this */ if (buf_len > mac_get_mtusize(ksci->ksci_machandle)) { @@ -219,6 +235,8 @@ kscimacnal_send(nal_cb_t *nal, return -ENOMEM; } + ktx->ktx_nmapped = 0; /* Start with no mapped pages :) */ + /* *SIGH* hdr is a stack variable in the calling function, so we * need to copy it to a buffer. Zerocopy magic (or is it just * deferred memcpy?) is annoying sometimes. */ @@ -235,19 +253,34 @@ kscimacnal_send(nal_cb_t *nal, lastblk=msg; /* Allocate additional mblks for each iov as needed. - * Essentially lib_copy_iov2buf with a twist or two */ + * Essentially lib_copy_(k)iov2buf with a twist or two */ while (payload_len > 0) { - ptl_size_t nob; + ptl_size_t nob; + char *addr; LASSERT (payload_niov > 0); - nob = MIN (payload_iov->iov_len, payload_len); + if(payload_iov != NULL) { + nob = MIN (payload_iov->iov_len, payload_len); + addr = payload_iov->iov_base; + } + else { + nob = MIN (payload_kiov->kiov_len, payload_len); + /* Bollocks. We need to handle paged IO for things to + * work but there is no good way to do this. We + * do it by kmap():ing all pages and keep them + * mapped until scimac is done with them. */ + /* FIXME: kunmap() on error */ + addr = kmap(payload_kiov->kiov_page); + ktx->ktx_kpages[ktx->ktx_nmapped++] = + payload_kiov->kiov_page; + } + /* We don't need a callback on the additional mblks, + * since all release callbacks seems to be called when + * the entire message has been sent */ + newblk=mac_alloc_mblk(addr, nob, NULL, NULL); - /* We don't need a callback on the additional mblks, since - * all release callbacks seems to be called when the entire - * message has been sent */ - newblk=mac_alloc_mblk(payload_iov->iov_base, nob, NULL, NULL); if(!newblk) { mac_free_msg(msg); PORTAL_FREE(ktx, (sizeof(kscimacnal_tx_t))); @@ -259,9 +292,16 @@ kscimacnal_send(nal_cb_t *nal, payload_len -= nob; payload_niov--; - payload_iov++; + if(payload_iov != NULL) { + payload_iov++; + } + else { + payload_kiov++; + } } + CDEBUG(D_NET, "kmapped %d pages\n", ktx->ktx_nmapped); + ktx->ktx_nal = nal; ktx->ktx_private = private; ktx->ktx_cookie = cookie; @@ -282,6 +322,39 @@ kscimacnal_send(nal_cb_t *nal, } +static int +kscimacnal_send (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_nob) +{ + return (kscimacnal_sendmsg (nal, private, cookie, hdr, type, nid, pid, + payload_niov, payload_iov, NULL, payload_nob)); +} + +static int +kscimacnal_send_pages (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_nob) +{ + return (kscimacnal_sendmsg (nal, private, cookie, hdr, type, nid, pid, + payload_niov, NULL, payload_kiov, payload_nob)); +} + + void kscimacnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd) { @@ -366,19 +439,22 @@ kscimacnal_rx(mac_handle_t *handle, mac_mblk_t *msg, mac_msg_type_t type, /* Called by portals to process a recieved packet */ -static int kscimacnal_recv(nal_cb_t *nal, - void *private, - lib_msg_t *cookie, - unsigned int niov, - struct iovec *iov, - size_t mlen, - size_t rlen) +inline static int +kscimacnal_recvmsg(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + struct iovec *iov, + ptl_kiov_t *kiov, + size_t mlen, + size_t rlen) { kscimacnal_rx_t *krx = private; mac_mblk_t *mblk; void *src; mac_size_t pkt_len; ptl_size_t iovused=0; + char *base=NULL; LASSERT (krx != NULL); LASSERT (krx->msg != NULL); @@ -393,6 +469,10 @@ static int kscimacnal_recv(nal_cb_t *nal, */ LASSERT (mlen==0 || mac_msg_size(krx->msg) >= sizeof(ptl_hdr_t)+rlen); LASSERT (mlen==0 || mlen <= rlen); + /* It must be OK to kmap() if required */ + LASSERT (kiov == NULL || !in_interrupt ()); + /* Either all pages or all vaddrs */ + LASSERT (!(kiov != NULL && iov != NULL)); PROF_START(memcpy); @@ -407,36 +487,59 @@ static int kscimacnal_recv(nal_cb_t *nal, LASSERT(src != NULL); - /* Essentially lib_copy_buf2iov but with continuation support, - * we "gracefully" thrash the argument vars ;) */ + /* Essentially lib_copy_buf2(k)iov but with continuation + * support, we "gracefully" thrash the argument vars ;) */ while (pkt_len > 0) { - ptl_size_t nob; + ptl_size_t nob, len; LASSERT (niov > 0); - LASSERT(iovused < iov->iov_len); + if(iov != NULL) { + LASSERT(iovused < iov->iov_len); + len = iov->iov_len; + base = iov->iov_base; + } + else { + LASSERT(iovused < kiov->kiov_len); + len = kiov->kiov_len; + if(base==NULL) { + /* New page */ + base = kmap(kiov->kiov_page); + } + } - nob = MIN (iov->iov_len-iovused, pkt_len); - CDEBUG(D_NET, "iovbase: %p iovlen: %d src: %p nob: %d " + nob = MIN (len-iovused, pkt_len); + CDEBUG(D_NET, "base: %p len: %d src: %p nob: %d " "iovused: %d\n", - iov->iov_base, iov->iov_len, - src, nob, iovused); + base, len, src, nob, iovused); - memcpy (iov->iov_base+iovused, src, nob); + memcpy (base+iovused, src, nob); pkt_len -= nob; src += nob; - if(nob+iovused < iov->iov_len) { + if(nob+iovused < len) { /* We didn't use all of the iov */ iovused+=nob; } else { niov--; - iov++; iovused=0; + if(iov != NULL) { + iov++; + } + else { + kunmap(kiov->kiov_page); + base=NULL; + kiov++; + } } } } + /* Just to make sure the last page is unmapped */ + if(kiov!=NULL && base!=NULL) { + kunmap(kiov->kiov_page); + base=NULL; + } PROF_FINISH(memcpy); CDEBUG(D_NET, "Calling lib_finalize.\n"); @@ -451,12 +554,38 @@ static int kscimacnal_recv(nal_cb_t *nal, } +static int +kscimacnal_recv(nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + struct iovec *iov, + size_t mlen, + size_t rlen) +{ + return (kscimacnal_recvmsg (nal, private, cookie, niov, iov, NULL, mlen, rlen)); +} + + +static int +kscimacnal_recv_pages (nal_cb_t *nal, + void *private, + lib_msg_t *cookie, + unsigned int niov, + ptl_kiov_t *kiov, + size_t mlen, + size_t rlen) +{ + return (kscimacnal_recvmsg (nal, private, cookie, niov, NULL, kiov, mlen, rlen)); +} + + nal_cb_t kscimacnal_lib = { nal_data: &kscimacnal_data, /* NAL private data */ cb_send: kscimacnal_send, - cb_send_pages: NULL, /* Ignore for now */ + cb_send_pages: kscimacnal_send_pages, cb_recv: kscimacnal_recv, - cb_recv_pages: NULL, + cb_recv_pages: kscimacnal_recv_pages, cb_read: kscimacnal_read, cb_write: kscimacnal_write, cb_malloc: kscimacnal_malloc, diff --git a/lustre/portals/libcfs/debug.c b/lustre/portals/libcfs/debug.c index 9d15c8f..e9b0e12 100644 --- a/lustre/portals/libcfs/debug.c +++ b/lustre/portals/libcfs/debug.c @@ -4,19 +4,19 @@ * Copyright (C) 2002 Cluster File Systems, Inc. * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/libcfs/module.c b/lustre/portals/libcfs/module.c index 9bb7e00..7ecbe43 100644 --- a/lustre/portals/libcfs/module.c +++ b/lustre/portals/libcfs/module.c @@ -3,19 +3,19 @@ * * Copyright (C) 2001, 2002 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/libcfs/proc.c b/lustre/portals/libcfs/proc.c index bb6c377..8817ace 100644 --- a/lustre/portals/libcfs/proc.c +++ b/lustre/portals/libcfs/proc.c @@ -6,19 +6,19 @@ * Author: Peter J. Braam * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/address.c b/lustre/portals/unals/address.c index b422c3f..6507924 100644 --- a/lustre/portals/unals/address.c +++ b/lustre/portals/unals/address.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/connection.c b/lustre/portals/unals/connection.c index 1d589bf..efc8a54 100644 --- a/lustre/portals/unals/connection.c +++ b/lustre/portals/unals/connection.c @@ -3,19 +3,19 @@ * * Copyright (c) 2002 Cray Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/debug.c b/lustre/portals/unals/debug.c index b73f042..b82bb2f 100644 --- a/lustre/portals/unals/debug.c +++ b/lustre/portals/unals/debug.c @@ -4,19 +4,19 @@ * Copyright (C) 2002 Cluster File Systems, Inc. * Author: Phil Schwan * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/pqtimer.c b/lustre/portals/unals/pqtimer.c index fa2fb4f..98c48eb 100644 --- a/lustre/portals/unals/pqtimer.c +++ b/lustre/portals/unals/pqtimer.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/procapi.c b/lustre/portals/unals/procapi.c index 4278822..206aea0 100644 --- a/lustre/portals/unals/procapi.c +++ b/lustre/portals/unals/procapi.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/proclib.c b/lustre/portals/unals/proclib.c index dcd64ed..2424145 100644 --- a/lustre/portals/unals/proclib.c +++ b/lustre/portals/unals/proclib.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/select.c b/lustre/portals/unals/select.c index 1dd547a..fe24efc 100644 --- a/lustre/portals/unals/select.c +++ b/lustre/portals/unals/select.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/table.c b/lustre/portals/unals/table.c index bef13c5..662775a 100644 --- a/lustre/portals/unals/table.c +++ b/lustre/portals/unals/table.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2002 Eric Hoffman * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/portals/unals/tcpnal.c b/lustre/portals/unals/tcpnal.c index f14a1fc..ec67a21 100644 --- a/lustre/portals/unals/tcpnal.c +++ b/lustre/portals/unals/tcpnal.c @@ -4,19 +4,19 @@ * Copyright (c) 2002 Cray Inc. * Copyright (c) 2003 Cluster File Systems, Inc. * - * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * This file is part of Lustre, http://www.lustre.org. * - * Portals is free software; you can redistribute it and/or - * modify it under the terms of version 2.1 of the GNU Lesser General - * Public License as published by the Free Software Foundation. + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. * - * Portals is distributed in the hope that it will be useful, + * Lustre is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. + * GNU General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with Portals; if not, write to the Free Software + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c new file mode 100644 index 0000000..650b65a --- /dev/null +++ b/lustre/ptlrpc/import.c @@ -0,0 +1,204 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (c) 2002, 2003 Cluster File Systems, Inc. + * Author: Mike Shaver + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#define DEBUG_SUBSYSTEM S_RPC +#ifdef __KERNEL__ +# include +# include +# include +#else +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ptlrpc_internal.h" + +/* should this take an imp_sem to ensure connect is single threaded? */ +int ptlrpc_connect_import(struct obd_import *imp) +{ + struct obd_device *obd = imp->imp_obd; + int msg_flags; + int initial_connect = 0; + int rc; + __u64 committed_before_reconnect = 0; + struct ptlrpc_request *request; + struct lustre_handle old_hdl; + int size[] = {sizeof(imp->imp_target_uuid), + sizeof(obd->obd_uuid), + sizeof(imp->imp_dlm_handle)}; + char *tmp[] = {imp->imp_target_uuid.uuid, + obd->obd_uuid.uuid, + (char *)&imp->imp_dlm_handle}; + unsigned long flags; + + spin_lock_irqsave(&imp->imp_lock, flags); + if (imp->imp_state == LUSTRE_IMP_CONNECTING) { + spin_unlock_irqrestore(&imp->imp_lock, flags); + RETURN(-EALREADY); + } else { + LASSERT(imp->imp_state == LUSTRE_IMP_DISCON); + } + CDEBUG(D_HA, "%s: new state: CONNECTING\n", + imp->imp_client->cli_name); + imp->imp_state = LUSTRE_IMP_CONNECTING; + imp->imp_conn_cnt++; + if (imp->imp_remote_handle.cookie == 0) { + initial_connect = 1; + } else { + committed_before_reconnect = imp->imp_peer_committed_transno; + } + spin_unlock_irqrestore(&imp->imp_lock, flags); + + request = ptlrpc_prep_req(imp, imp->imp_connect_op, 3, size, tmp); + if (!request) + GOTO(out, rc = -ENOMEM); + + request->rq_send_state = LUSTRE_IMP_CONNECTING; + request->rq_replen = lustre_msg_size(0, NULL); + + // lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_PEER); + + rc = ptlrpc_queue_wait(request); + if (rc) { + GOTO(free_req, rc); + } + + msg_flags = lustre_msg_get_op_flags(request->rq_repmsg); + + if (initial_connect) { + CDEBUG(D_HA, "%s: new state: FULL\n", + imp->imp_client->cli_name); + if (msg_flags & MSG_CONNECT_REPLAYABLE) + imp->imp_replayable = 1; + imp->imp_remote_handle = request->rq_repmsg->handle; + imp->imp_state = LUSTRE_IMP_FULL; + GOTO(free_req, rc = 0); + } + + /* Determine what recovery state to move the import to. */ + if (MSG_CONNECT_RECONNECT & msg_flags) { + memset(&old_hdl, 0, sizeof(old_hdl)); + if (!memcmp(&old_hdl, &request->rq_repmsg->handle, + sizeof (old_hdl))) { + CERROR("%s@%s didn't like our handle "LPX64 + ", failed\n", imp->imp_target_uuid.uuid, + imp->imp_connection->c_remote_uuid.uuid, + imp->imp_dlm_handle.cookie); + GOTO(free_req, rc = -ENOTCONN); + } + + if (memcmp(&imp->imp_remote_handle, &request->rq_repmsg->handle, + sizeof(imp->imp_remote_handle))) { + CERROR("%s@%s changed handle from "LPX64" to "LPX64 + "; copying, but this may foreshadow disaster\n", + imp->imp_target_uuid.uuid, + imp->imp_connection->c_remote_uuid.uuid, + imp->imp_remote_handle.cookie, + request->rq_repmsg->handle.cookie); + imp->imp_remote_handle = request->rq_repmsg->handle; + } else { + CERROR("reconnected to %s@%s after partition\n", + imp->imp_target_uuid.uuid, + imp->imp_connection->c_remote_uuid.uuid); + } + CDEBUG(D_HA, "%s: new state: RECOVER\n", + imp->imp_client->cli_name); + imp->imp_state = LUSTRE_IMP_RECOVER; + } + else if (MSG_CONNECT_RECOVERING & msg_flags) { + CDEBUG(D_HA, "%s: new state: REPLAY\n", + imp->imp_client->cli_name); + LASSERT(imp->imp_replayable); + imp->imp_state = LUSTRE_IMP_RECOVER; + imp->imp_remote_handle = request->rq_repmsg->handle; + imp->imp_state = LUSTRE_IMP_REPLAY; + } + else { + CDEBUG(D_HA, "%s: new state: EVICTED\n", + imp->imp_client->cli_name); + imp->imp_remote_handle = request->rq_repmsg->handle; + imp->imp_state = LUSTRE_IMP_EVICTED; + } + + /* Sanity checks for a reconnected import. */ + if (!(imp->imp_replayable) != + !(msg_flags & MSG_CONNECT_REPLAYABLE)) { + CERROR("imp_replayable flag does not match server " + "after reconnect. We should LBUG right here.\n"); + } + + if (request->rq_repmsg->last_committed < committed_before_reconnect) { + CERROR("%s went back in time (transno "LPD64 + " was previously committed, server now claims "LPD64 + ")! is shared storage not coherent?\n", + imp->imp_target_uuid.uuid, + committed_before_reconnect, + request->rq_repmsg->last_committed); + } + + free_req: + ptlrpc_req_finished(request); + + out: + if (rc != 0) + imp->imp_state = LUSTRE_IMP_DISCON; + RETURN(rc); +} + + + +int ptlrpc_disconnect_import(struct obd_import *imp) +{ + struct ptlrpc_request *request; + int rq_opc; + int rc = 0; + ENTRY; + + switch (imp->imp_connect_op) { + case OST_CONNECT: rq_opc = OST_DISCONNECT; break; + case MDS_CONNECT: rq_opc = MDS_DISCONNECT; break; + case MGMT_CONNECT:rq_opc = MGMT_DISCONNECT;break; + default: + CERROR("don't know how to disconnect from %s (connect_op %d)\n", + imp->imp_target_uuid.uuid, imp->imp_connect_op); + RETURN(-EINVAL); + } + + request = ptlrpc_prep_req(imp, rq_opc, 0, NULL, NULL); + if (request) { + request->rq_replen = lustre_msg_size(0, NULL); + rc = ptlrpc_queue_wait(request); + ptlrpc_req_finished(request); + } + + imp->imp_state = LUSTRE_IMP_DISCON; + memset(&imp->imp_remote_handle, 0, sizeof(imp->imp_remote_handle)); + RETURN(rc); +} + -- 1.8.3.1