From 0073a99dbd25bdb046d6bf11745bc0440930eb9a Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 18 Nov 2004 17:37:20 +0000 Subject: [PATCH] * Added lonal (loopback NAL) --- lnet/include/linux/kp30.h | 1 + lnet/include/linux/libcfs.h | 1 + lnet/klnds/Makefile.in | 1 + lnet/klnds/Makefile.mk | 1 + lnet/klnds/autoMakefile.am | 2 +- lnet/klnds/lolnd/.cvsignore | 10 ++ lnet/klnds/lolnd/Makefile.in | 4 + lnet/klnds/lolnd/autoMakefile.am | 11 ++ lnet/klnds/lolnd/lolnd.c | 167 ++++++++++++++++++ lnet/klnds/lolnd/lolnd.h | 77 ++++++++ lnet/klnds/lolnd/lolnd_cb.c | 272 +++++++++++++++++++++++++++++ lnet/utils/portals.c | 1 + lustre/configure.in | 2 + lustre/portals/include/linux/kp30.h | 1 + lustre/portals/include/linux/libcfs.h | 1 + lustre/portals/knals/Makefile.in | 1 + lustre/portals/knals/Makefile.mk | 1 + lustre/portals/knals/autoMakefile.am | 2 +- lustre/portals/knals/lonal/.cvsignore | 10 ++ lustre/portals/knals/lonal/Makefile.in | 4 + lustre/portals/knals/lonal/autoMakefile.am | 11 ++ lustre/portals/knals/lonal/lonal.c | 167 ++++++++++++++++++ lustre/portals/knals/lonal/lonal.h | 77 ++++++++ lustre/portals/knals/lonal/lonal_cb.c | 272 +++++++++++++++++++++++++++++ lustre/portals/utils/portals.c | 1 + lustre/ptlrpc/events.c | 12 +- lustre/utils/lconf | 5 + lustre/utils/lmc | 2 +- 28 files changed, 1109 insertions(+), 8 deletions(-) create mode 100644 lnet/klnds/lolnd/.cvsignore create mode 100644 lnet/klnds/lolnd/Makefile.in create mode 100644 lnet/klnds/lolnd/autoMakefile.am create mode 100644 lnet/klnds/lolnd/lolnd.c create mode 100644 lnet/klnds/lolnd/lolnd.h create mode 100644 lnet/klnds/lolnd/lolnd_cb.c create mode 100644 lustre/portals/knals/lonal/.cvsignore create mode 100644 lustre/portals/knals/lonal/Makefile.in create mode 100644 lustre/portals/knals/lonal/autoMakefile.am create mode 100644 lustre/portals/knals/lonal/lonal.c create mode 100644 lustre/portals/knals/lonal/lonal.h create mode 100644 lustre/portals/knals/lonal/lonal_cb.c diff --git a/lnet/include/linux/kp30.h b/lnet/include/linux/kp30.h index 1ec69d5..9006f13 100644 --- a/lnet/include/linux/kp30.h +++ b/lnet/include/linux/kp30.h @@ -653,6 +653,7 @@ enum { ROUTER = 6, OPENIBNAL = 7, IIBNAL = 8, + LONAL = 9, NAL_ENUM_END_MARKER }; diff --git a/lnet/include/linux/libcfs.h b/lnet/include/linux/libcfs.h index 8317f14..f6a0a60 100644 --- a/lnet/include/linux/libcfs.h +++ b/lnet/include/linux/libcfs.h @@ -110,6 +110,7 @@ struct ptldebug_header { #define S_ASOBD 0x01000000 #define S_LMV 0x02000000 #define S_CMOBD 0x04000000 +#define S_LONAL 0x08000000 /* If you change these values, please keep portals/utils/debug.c * up to date! */ diff --git a/lnet/klnds/Makefile.in b/lnet/klnds/Makefile.in index 9763d14..610b55c 100644 --- a/lnet/klnds/Makefile.in +++ b/lnet/klnds/Makefile.in @@ -3,5 +3,6 @@ @BUILD_IIBNAL_TRUE@subdir-m += iibnal @BUILD_QSWNAL_TRUE@subdir-m += qswnal subdir-m += socknal +subdir-m += lonal @INCLUDE_RULES@ diff --git a/lnet/klnds/Makefile.mk b/lnet/klnds/Makefile.mk index cd5d9d6..454ee16 100644 --- a/lnet/klnds/Makefile.mk +++ b/lnet/klnds/Makefile.mk @@ -1,4 +1,5 @@ include $(obj)/../Kernelenv obj-y = socknal/ +obj-y = lonal/ # more coming... diff --git a/lnet/klnds/autoMakefile.am b/lnet/klnds/autoMakefile.am index 0090364..2bc4b5c 100644 --- a/lnet/klnds/autoMakefile.am +++ b/lnet/klnds/autoMakefile.am @@ -3,4 +3,4 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution -SUBDIRS = gmnal iibnal openibnal qswnal socknal +SUBDIRS = gmnal iibnal openibnal qswnal socknal lonal diff --git a/lnet/klnds/lolnd/.cvsignore b/lnet/klnds/lolnd/.cvsignore new file mode 100644 index 0000000..5ed596b --- /dev/null +++ b/lnet/klnds/lolnd/.cvsignore @@ -0,0 +1,10 @@ +.deps +Makefile +.*.cmd +autoMakefile.in +autoMakefile +*.ko +*.mod.c +.*.flags +.tmp_versions +.depend diff --git a/lnet/klnds/lolnd/Makefile.in b/lnet/klnds/lolnd/Makefile.in new file mode 100644 index 0000000..222e861 --- /dev/null +++ b/lnet/klnds/lolnd/Makefile.in @@ -0,0 +1,4 @@ +MODULES := klonal +klonal-objs := lonal.o lonal_cb.o + +@INCLUDE_RULES@ diff --git a/lnet/klnds/lolnd/autoMakefile.am b/lnet/klnds/lolnd/autoMakefile.am new file mode 100644 index 0000000..d1ef995 --- /dev/null +++ b/lnet/klnds/lolnd/autoMakefile.am @@ -0,0 +1,11 @@ +# Copyright (C) 2001 Cluster File Systems, Inc. +# +# This code is issued under the GNU General Public License. +# See the file COPYING in this distribution + +if MODULES +modulenet_DATA = klonal$(KMODEXT) +endif + +MOSTLYCLEANFILES = *.o *.ko *.mod.c +DIST_SOURCES = $(klonal-objs:%.o=%.c) lonal.h diff --git a/lnet/klnds/lolnd/lolnd.c b/lnet/klnds/lolnd/lolnd.c new file mode 100644 index 0000000..56f4a77 --- /dev/null +++ b/lnet/klnds/lolnd/lolnd.c @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/lustre/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +nal_t klonal_api; +klonal_data_t klonal_data; +ptl_handle_ni_t klonal_ni; + + +int +klonal_cmd (struct portals_cfg *pcfg, void *private) +{ + LASSERT (pcfg != NULL); + + switch (pcfg->pcfg_command) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID to "LPX64" (was "LPX64")\n", + pcfg->pcfg_nid, klonal_lib.libnal_ni.ni_pid.nid); + klonal_lib.libnal_ni.ni_pid.nid = pcfg->pcfg_nid; + return (0); + + default: + return (-EINVAL); + } +} + +static void +klonal_shutdown(nal_t *nal) +{ + /* NB The first ref was this module! */ + if (nal->nal_refct != 0) + return; + + CDEBUG (D_NET, "shutdown\n"); + LASSERT (nal == &klonal_api); + + switch (klonal_data.klo_init) + { + default: + LASSERT (0); + + case KLO_INIT_ALL: + libcfs_nal_cmd_unregister(LONAL); + /* fall through */ + + case KLO_INIT_LIB: + lib_fini (&klonal_lib); + break; + + case KLO_INIT_NOTHING: + return; + } + + memset(&klonal_data, 0, sizeof (klonal_data)); + + CDEBUG (D_MALLOC, "done kmem %d\n", atomic_read(&portal_kmemory)); + + printk (KERN_INFO "Lustre: LO NAL unloaded (final mem %d)\n", + atomic_read(&portal_kmemory)); + PORTAL_MODULE_UNUSE; +} + +static int +klonal_startup (nal_t *nal, ptl_pid_t requested_pid, + ptl_ni_limits_t *requested_limits, + ptl_ni_limits_t *actual_limits) +{ + int rc; + ptl_process_id_t my_process_id; + int pkmem = atomic_read(&portal_kmemory); + + LASSERT (nal == &klonal_api); + + if (nal->nal_refct != 0) { + if (actual_limits != NULL) + *actual_limits = klonal_lib.libnal_ni.ni_actual_limits; + return (PTL_OK); + } + + LASSERT (klonal_data.klo_init == KLO_INIT_NOTHING); + + CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory)); + + /* ensure all pointers NULL etc */ + memset (&klonal_data, 0, sizeof (klonal_data)); + + my_process_id.nid = 0; + my_process_id.pid = requested_pid; + + rc = lib_init(&klonal_lib, nal, my_process_id, + requested_limits, actual_limits); + if (rc != PTL_OK) { + CERROR ("lib_init failed %d\n", rc); + klonal_shutdown (nal); + return (rc); + } + + klonal_data.klo_init = KLO_INIT_LIB; + + rc = libcfs_nal_cmd_register (LONAL, &klonal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + klonal_shutdown (nal); + return (PTL_FAIL); + } + + klonal_data.klo_init = KLO_INIT_ALL; + + printk(KERN_INFO "Lustre: LO NAL (initial mem %d)\n", pkmem); + PORTAL_MODULE_USE; + + return (PTL_OK); +} + +void __exit +klonal_finalise (void) +{ + PtlNIFini(klonal_ni); + + ptl_unregister_nal(LONAL); +} + +static int __init +klonal_initialise (void) +{ + int rc; + + klonal_api.nal_ni_init = klonal_startup; + klonal_api.nal_ni_fini = klonal_shutdown; + + rc = ptl_register_nal(LONAL, &klonal_api); + if (rc != PTL_OK) { + CERROR("Can't register LONAL: %d\n", rc); + return (-ENOMEM); /* or something... */ + } + + return (0); +} + +MODULE_AUTHOR("Cluster File Systems, Inc. "); +MODULE_DESCRIPTION("Loopback NAL v0.01"); +MODULE_LICENSE("GPL"); + +module_init (klonal_initialise); +module_exit (klonal_finalise); diff --git a/lnet/klnds/lolnd/lolnd.h b/lnet/klnds/lolnd/lolnd.h new file mode 100644 index 0000000..f1675a7 --- /dev/null +++ b/lnet/klnds/lolnd/lolnd.h @@ -0,0 +1,77 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Basic library routines. + * + */ + +#ifndef _LONAL_H +#define _LONAL_H +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_SUBSYSTEM S_LONAL + +#include +#include +#include +#include + +#define KLOD_IOV 153401 +#define KLOD_KIOV 153402 + +typedef struct +{ + unsigned int klod_type; + unsigned int klod_niov; + size_t klod_offset; + size_t klod_nob; + union { + struct iovec *iov; + ptl_kiov_t *kiov; + } klod_iov; + +} klo_desc_t; + +typedef struct +{ + char klo_init; /* what's been initialised */ +} klonal_data_t; + +/* kqn_init state */ +#define KLO_INIT_NOTHING 0 /* MUST BE ZERO so zeroed state is initialised OK */ +#define KLO_INIT_LIB 1 +#define KLO_INIT_ALL 2 + +extern lib_nal_t klonal_lib; +extern nal_t klonal_api; +extern klonal_data_t klonal_data; + +#endif /* _LONAL_H */ diff --git a/lnet/klnds/lolnd/lolnd_cb.c b/lnet/klnds/lolnd/lolnd_cb.c new file mode 100644 index 0000000..dccc81a --- /dev/null +++ b/lnet/klnds/lolnd/lolnd_cb.c @@ -0,0 +1,272 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +/* + * LIB functions follow + * + */ +static int +klonal_dist(lib_nal_t *nal, ptl_nid_t nid, unsigned long *dist) +{ + *dist = 0; /* it's me */ + return (0); +} + +static ptl_err_t +klonal_send (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_IOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.iov = payload_iov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_send_pages (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_KIOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.kiov = payload_kiov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_recv(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t mlen, + size_t rlen) +{ + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle mapped->mapped matches */ + LASSERT(klod->klod_type == KLOD_IOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= iov->iov_len) { + offset -= iov->iov_len; + iov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.iov->iov_len) { + klod->klod_offset -= klod->klod_iov.iov->iov_len; + klod->klod_iov.iov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + int fraglen = MIN(iov->iov_len - offset, + klod->klod_iov.iov->iov_len - klod->klod_offset); + + LASSERT(niov > 0); + LASSERT(klod->klod_niov > 0); + + if (fraglen > mlen) + fraglen = mlen; + + memcpy((void *)((unsigned long)iov->iov_base + offset), + (void *)((unsigned long)klod->klod_iov.iov->iov_base + + klod->klod_offset), + fraglen); + + if (offset + fraglen < iov->iov_len) { + offset += fraglen; + } else { + offset = 0; + iov++; + niov--; + } + + if (klod->klod_offset + fraglen < klod->klod_iov.iov->iov_len ) { + klod->klod_offset += fraglen; + } else { + klod->klod_offset = 0; + klod->klod_iov.iov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +static ptl_err_t +klonal_recv_pages(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + ptl_kiov_t *kiov, + size_t offset, + size_t mlen, + size_t rlen) +{ + void *srcaddr = NULL; + void *dstaddr = NULL; + unsigned long srcfrag = 0; + unsigned long dstfrag = 0; + unsigned long fraglen; + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle unmapped->unmapped matches */ + LASSERT(klod->klod_type == KLOD_KIOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= kiov->kiov_len) { + offset -= kiov->kiov_len; + kiov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.kiov->kiov_len) { + klod->klod_offset -= klod->klod_iov.kiov->kiov_len; + klod->klod_iov.kiov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + /* CAVEAT EMPTOR: I kmap 2 pages at once == slight risk of deadlock */ + LASSERT(niov > 0); + if (dstaddr == NULL) { + dstaddr = (void *)((unsigned long)kmap(kiov->kiov_page) + + kiov->kiov_offset + offset); + dstfrag = kiov->kiov_len - offset; + } + + LASSERT(klod->klod_niov > 0); + if (srcaddr == NULL) { + srcaddr = (void *)((unsigned long)kmap(klod->klod_iov.kiov->kiov_page) + + klod->klod_iov.kiov->kiov_offset + klod->klod_offset); + srcfrag = klod->klod_iov.kiov->kiov_len - klod->klod_offset; + } + + fraglen = MIN(srcfrag, dstfrag); + if (fraglen > mlen) + fraglen = mlen; + + memcpy(dstaddr, srcaddr, fraglen); + + if (fraglen < dstfrag) { + dstfrag -= fraglen; + dstaddr = (void *)((unsigned long)dstaddr + fraglen); + } else { + kunmap(kiov->kiov_page); + dstaddr = NULL; + offset = 0; + kiov++; + niov--; + } + + if (fraglen < srcfrag) { + srcfrag -= fraglen; + srcaddr = (void *)((unsigned long)srcaddr + fraglen); + } else { + kunmap(klod->klod_iov.kiov->kiov_page); + srcaddr = NULL; + klod->klod_offset = 0; + klod->klod_iov.kiov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + if (dstaddr != NULL) + kunmap(kiov->kiov_page); + + if (srcaddr != NULL) + kunmap(klod->klod_iov.kiov->kiov_page); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +lib_nal_t klonal_lib = +{ + libnal_data: &klonal_data, /* NAL private data */ + libnal_send: klonal_send, + libnal_send_pages: klonal_send_pages, + libnal_recv: klonal_recv, + libnal_recv_pages: klonal_recv_pages, + libnal_dist: klonal_dist +}; diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c index d5d29dc..07fada2 100644 --- a/lnet/utils/portals.c +++ b/lnet/utils/portals.c @@ -71,6 +71,7 @@ static name2num_t nalnames[] = { {"gm", GMNAL}, {"openib", OPENIBNAL}, {"iib", IIBNAL}, + {"lo", LONAL}, {NULL, -1} }; diff --git a/lustre/configure.in b/lustre/configure.in index fe97687..b0f9861 100644 --- a/lustre/configure.in +++ b/lustre/configure.in @@ -226,6 +226,8 @@ portals/knals/qswnal/Makefile portals/knals/qswnal/autoMakefile portals/knals/socknal/Makefile portals/knals/socknal/autoMakefile +portals/knals/lonal/Makefile +portals/knals/lonal/autoMakefile portals/libcfs/Makefile portals/libcfs/autoMakefile portals/portals/Makefile diff --git a/lustre/portals/include/linux/kp30.h b/lustre/portals/include/linux/kp30.h index 1ec69d5..9006f13 100644 --- a/lustre/portals/include/linux/kp30.h +++ b/lustre/portals/include/linux/kp30.h @@ -653,6 +653,7 @@ enum { ROUTER = 6, OPENIBNAL = 7, IIBNAL = 8, + LONAL = 9, NAL_ENUM_END_MARKER }; diff --git a/lustre/portals/include/linux/libcfs.h b/lustre/portals/include/linux/libcfs.h index 8317f14..f6a0a60 100644 --- a/lustre/portals/include/linux/libcfs.h +++ b/lustre/portals/include/linux/libcfs.h @@ -110,6 +110,7 @@ struct ptldebug_header { #define S_ASOBD 0x01000000 #define S_LMV 0x02000000 #define S_CMOBD 0x04000000 +#define S_LONAL 0x08000000 /* If you change these values, please keep portals/utils/debug.c * up to date! */ diff --git a/lustre/portals/knals/Makefile.in b/lustre/portals/knals/Makefile.in index 9763d14..610b55c 100644 --- a/lustre/portals/knals/Makefile.in +++ b/lustre/portals/knals/Makefile.in @@ -3,5 +3,6 @@ @BUILD_IIBNAL_TRUE@subdir-m += iibnal @BUILD_QSWNAL_TRUE@subdir-m += qswnal subdir-m += socknal +subdir-m += lonal @INCLUDE_RULES@ diff --git a/lustre/portals/knals/Makefile.mk b/lustre/portals/knals/Makefile.mk index cd5d9d6..454ee16 100644 --- a/lustre/portals/knals/Makefile.mk +++ b/lustre/portals/knals/Makefile.mk @@ -1,4 +1,5 @@ include $(obj)/../Kernelenv obj-y = socknal/ +obj-y = lonal/ # more coming... diff --git a/lustre/portals/knals/autoMakefile.am b/lustre/portals/knals/autoMakefile.am index 0090364..2bc4b5c 100644 --- a/lustre/portals/knals/autoMakefile.am +++ b/lustre/portals/knals/autoMakefile.am @@ -3,4 +3,4 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution -SUBDIRS = gmnal iibnal openibnal qswnal socknal +SUBDIRS = gmnal iibnal openibnal qswnal socknal lonal diff --git a/lustre/portals/knals/lonal/.cvsignore b/lustre/portals/knals/lonal/.cvsignore new file mode 100644 index 0000000..5ed596b --- /dev/null +++ b/lustre/portals/knals/lonal/.cvsignore @@ -0,0 +1,10 @@ +.deps +Makefile +.*.cmd +autoMakefile.in +autoMakefile +*.ko +*.mod.c +.*.flags +.tmp_versions +.depend diff --git a/lustre/portals/knals/lonal/Makefile.in b/lustre/portals/knals/lonal/Makefile.in new file mode 100644 index 0000000..222e861 --- /dev/null +++ b/lustre/portals/knals/lonal/Makefile.in @@ -0,0 +1,4 @@ +MODULES := klonal +klonal-objs := lonal.o lonal_cb.o + +@INCLUDE_RULES@ diff --git a/lustre/portals/knals/lonal/autoMakefile.am b/lustre/portals/knals/lonal/autoMakefile.am new file mode 100644 index 0000000..d1ef995 --- /dev/null +++ b/lustre/portals/knals/lonal/autoMakefile.am @@ -0,0 +1,11 @@ +# Copyright (C) 2001 Cluster File Systems, Inc. +# +# This code is issued under the GNU General Public License. +# See the file COPYING in this distribution + +if MODULES +modulenet_DATA = klonal$(KMODEXT) +endif + +MOSTLYCLEANFILES = *.o *.ko *.mod.c +DIST_SOURCES = $(klonal-objs:%.o=%.c) lonal.h diff --git a/lustre/portals/knals/lonal/lonal.c b/lustre/portals/knals/lonal/lonal.c new file mode 100644 index 0000000..56f4a77 --- /dev/null +++ b/lustre/portals/knals/lonal/lonal.c @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/lustre/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +nal_t klonal_api; +klonal_data_t klonal_data; +ptl_handle_ni_t klonal_ni; + + +int +klonal_cmd (struct portals_cfg *pcfg, void *private) +{ + LASSERT (pcfg != NULL); + + switch (pcfg->pcfg_command) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID to "LPX64" (was "LPX64")\n", + pcfg->pcfg_nid, klonal_lib.libnal_ni.ni_pid.nid); + klonal_lib.libnal_ni.ni_pid.nid = pcfg->pcfg_nid; + return (0); + + default: + return (-EINVAL); + } +} + +static void +klonal_shutdown(nal_t *nal) +{ + /* NB The first ref was this module! */ + if (nal->nal_refct != 0) + return; + + CDEBUG (D_NET, "shutdown\n"); + LASSERT (nal == &klonal_api); + + switch (klonal_data.klo_init) + { + default: + LASSERT (0); + + case KLO_INIT_ALL: + libcfs_nal_cmd_unregister(LONAL); + /* fall through */ + + case KLO_INIT_LIB: + lib_fini (&klonal_lib); + break; + + case KLO_INIT_NOTHING: + return; + } + + memset(&klonal_data, 0, sizeof (klonal_data)); + + CDEBUG (D_MALLOC, "done kmem %d\n", atomic_read(&portal_kmemory)); + + printk (KERN_INFO "Lustre: LO NAL unloaded (final mem %d)\n", + atomic_read(&portal_kmemory)); + PORTAL_MODULE_UNUSE; +} + +static int +klonal_startup (nal_t *nal, ptl_pid_t requested_pid, + ptl_ni_limits_t *requested_limits, + ptl_ni_limits_t *actual_limits) +{ + int rc; + ptl_process_id_t my_process_id; + int pkmem = atomic_read(&portal_kmemory); + + LASSERT (nal == &klonal_api); + + if (nal->nal_refct != 0) { + if (actual_limits != NULL) + *actual_limits = klonal_lib.libnal_ni.ni_actual_limits; + return (PTL_OK); + } + + LASSERT (klonal_data.klo_init == KLO_INIT_NOTHING); + + CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory)); + + /* ensure all pointers NULL etc */ + memset (&klonal_data, 0, sizeof (klonal_data)); + + my_process_id.nid = 0; + my_process_id.pid = requested_pid; + + rc = lib_init(&klonal_lib, nal, my_process_id, + requested_limits, actual_limits); + if (rc != PTL_OK) { + CERROR ("lib_init failed %d\n", rc); + klonal_shutdown (nal); + return (rc); + } + + klonal_data.klo_init = KLO_INIT_LIB; + + rc = libcfs_nal_cmd_register (LONAL, &klonal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + klonal_shutdown (nal); + return (PTL_FAIL); + } + + klonal_data.klo_init = KLO_INIT_ALL; + + printk(KERN_INFO "Lustre: LO NAL (initial mem %d)\n", pkmem); + PORTAL_MODULE_USE; + + return (PTL_OK); +} + +void __exit +klonal_finalise (void) +{ + PtlNIFini(klonal_ni); + + ptl_unregister_nal(LONAL); +} + +static int __init +klonal_initialise (void) +{ + int rc; + + klonal_api.nal_ni_init = klonal_startup; + klonal_api.nal_ni_fini = klonal_shutdown; + + rc = ptl_register_nal(LONAL, &klonal_api); + if (rc != PTL_OK) { + CERROR("Can't register LONAL: %d\n", rc); + return (-ENOMEM); /* or something... */ + } + + return (0); +} + +MODULE_AUTHOR("Cluster File Systems, Inc. "); +MODULE_DESCRIPTION("Loopback NAL v0.01"); +MODULE_LICENSE("GPL"); + +module_init (klonal_initialise); +module_exit (klonal_finalise); diff --git a/lustre/portals/knals/lonal/lonal.h b/lustre/portals/knals/lonal/lonal.h new file mode 100644 index 0000000..f1675a7 --- /dev/null +++ b/lustre/portals/knals/lonal/lonal.h @@ -0,0 +1,77 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Basic library routines. + * + */ + +#ifndef _LONAL_H +#define _LONAL_H +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_SUBSYSTEM S_LONAL + +#include +#include +#include +#include + +#define KLOD_IOV 153401 +#define KLOD_KIOV 153402 + +typedef struct +{ + unsigned int klod_type; + unsigned int klod_niov; + size_t klod_offset; + size_t klod_nob; + union { + struct iovec *iov; + ptl_kiov_t *kiov; + } klod_iov; + +} klo_desc_t; + +typedef struct +{ + char klo_init; /* what's been initialised */ +} klonal_data_t; + +/* kqn_init state */ +#define KLO_INIT_NOTHING 0 /* MUST BE ZERO so zeroed state is initialised OK */ +#define KLO_INIT_LIB 1 +#define KLO_INIT_ALL 2 + +extern lib_nal_t klonal_lib; +extern nal_t klonal_api; +extern klonal_data_t klonal_data; + +#endif /* _LONAL_H */ diff --git a/lustre/portals/knals/lonal/lonal_cb.c b/lustre/portals/knals/lonal/lonal_cb.c new file mode 100644 index 0000000..dccc81a --- /dev/null +++ b/lustre/portals/knals/lonal/lonal_cb.c @@ -0,0 +1,272 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +/* + * LIB functions follow + * + */ +static int +klonal_dist(lib_nal_t *nal, ptl_nid_t nid, unsigned long *dist) +{ + *dist = 0; /* it's me */ + return (0); +} + +static ptl_err_t +klonal_send (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_IOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.iov = payload_iov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_send_pages (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_KIOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.kiov = payload_kiov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_recv(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t mlen, + size_t rlen) +{ + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle mapped->mapped matches */ + LASSERT(klod->klod_type == KLOD_IOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= iov->iov_len) { + offset -= iov->iov_len; + iov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.iov->iov_len) { + klod->klod_offset -= klod->klod_iov.iov->iov_len; + klod->klod_iov.iov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + int fraglen = MIN(iov->iov_len - offset, + klod->klod_iov.iov->iov_len - klod->klod_offset); + + LASSERT(niov > 0); + LASSERT(klod->klod_niov > 0); + + if (fraglen > mlen) + fraglen = mlen; + + memcpy((void *)((unsigned long)iov->iov_base + offset), + (void *)((unsigned long)klod->klod_iov.iov->iov_base + + klod->klod_offset), + fraglen); + + if (offset + fraglen < iov->iov_len) { + offset += fraglen; + } else { + offset = 0; + iov++; + niov--; + } + + if (klod->klod_offset + fraglen < klod->klod_iov.iov->iov_len ) { + klod->klod_offset += fraglen; + } else { + klod->klod_offset = 0; + klod->klod_iov.iov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +static ptl_err_t +klonal_recv_pages(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + ptl_kiov_t *kiov, + size_t offset, + size_t mlen, + size_t rlen) +{ + void *srcaddr = NULL; + void *dstaddr = NULL; + unsigned long srcfrag = 0; + unsigned long dstfrag = 0; + unsigned long fraglen; + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle unmapped->unmapped matches */ + LASSERT(klod->klod_type == KLOD_KIOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= kiov->kiov_len) { + offset -= kiov->kiov_len; + kiov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.kiov->kiov_len) { + klod->klod_offset -= klod->klod_iov.kiov->kiov_len; + klod->klod_iov.kiov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + /* CAVEAT EMPTOR: I kmap 2 pages at once == slight risk of deadlock */ + LASSERT(niov > 0); + if (dstaddr == NULL) { + dstaddr = (void *)((unsigned long)kmap(kiov->kiov_page) + + kiov->kiov_offset + offset); + dstfrag = kiov->kiov_len - offset; + } + + LASSERT(klod->klod_niov > 0); + if (srcaddr == NULL) { + srcaddr = (void *)((unsigned long)kmap(klod->klod_iov.kiov->kiov_page) + + klod->klod_iov.kiov->kiov_offset + klod->klod_offset); + srcfrag = klod->klod_iov.kiov->kiov_len - klod->klod_offset; + } + + fraglen = MIN(srcfrag, dstfrag); + if (fraglen > mlen) + fraglen = mlen; + + memcpy(dstaddr, srcaddr, fraglen); + + if (fraglen < dstfrag) { + dstfrag -= fraglen; + dstaddr = (void *)((unsigned long)dstaddr + fraglen); + } else { + kunmap(kiov->kiov_page); + dstaddr = NULL; + offset = 0; + kiov++; + niov--; + } + + if (fraglen < srcfrag) { + srcfrag -= fraglen; + srcaddr = (void *)((unsigned long)srcaddr + fraglen); + } else { + kunmap(klod->klod_iov.kiov->kiov_page); + srcaddr = NULL; + klod->klod_offset = 0; + klod->klod_iov.kiov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + if (dstaddr != NULL) + kunmap(kiov->kiov_page); + + if (srcaddr != NULL) + kunmap(klod->klod_iov.kiov->kiov_page); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +lib_nal_t klonal_lib = +{ + libnal_data: &klonal_data, /* NAL private data */ + libnal_send: klonal_send, + libnal_send_pages: klonal_send_pages, + libnal_recv: klonal_recv, + libnal_recv_pages: klonal_recv_pages, + libnal_dist: klonal_dist +}; diff --git a/lustre/portals/utils/portals.c b/lustre/portals/utils/portals.c index d5d29dc..07fada2 100644 --- a/lustre/portals/utils/portals.c +++ b/lustre/portals/utils/portals.c @@ -71,6 +71,7 @@ static name2num_t nalnames[] = { {"gm", GMNAL}, {"openib", OPENIBNAL}, {"iib", IIBNAL}, + {"lo", LONAL}, {NULL, -1} }; diff --git a/lustre/ptlrpc/events.c b/lustre/ptlrpc/events.c index aae2bec..ac83a55 100644 --- a/lustre/ptlrpc/events.c +++ b/lustre/ptlrpc/events.c @@ -616,11 +616,13 @@ int ptlrpc_init_portals(void) char *name; } ptl_nis[] = { #ifndef CRAY_PORTALS - {QSWNAL, "qswnal"}, - {SOCKNAL, "socknal"}, - {GMNAL, "gmnal"}, - {OPENIBNAL, "openibnal"}, - {TCPNAL, "tcpnal"}, + {QSWNAL, "qswnal"}, + {SOCKNAL, "socknal"}, + {GMNAL, "gmnal"}, + {OPENIBNAL, "openibnal"}, + {IIBNAL, "iibnal"}, + {TCPNAL, "tcpnal"}, + {LONAL, "lonal"}, #else {CRAY_KB_ERNAL, "cray_kb_ernal"}, #endif diff --git a/lustre/utils/lconf b/lustre/utils/lconf index cd30cf5..22af909 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -120,6 +120,7 @@ subsystem_names = { "asobd" : (1 << 24), "lmv" : (1 << 25), "cmobd" : (1 << 26), + "lonal" : (1 << 27), } @@ -1077,6 +1078,8 @@ def sys_get_local_address(net_type, wildcard, cluster_id): local = elan_id except IOError, e: log(e) + elif net_type == 'lo': + fixme("automatic local address for loopback") elif net_type == 'gm': fixme("automatic local address for GM") @@ -1364,6 +1367,8 @@ class Network(Module): manager.add_portals_module("knals/openibnal", 'kopenibnal') if self.net_type == 'iib': manager.add_portals_module("knals/iibnal", 'kiibnal') + if self.net_type == 'lo': + manager.add_portals_module("knals/lonal", 'klonal') def nid_to_uuid(self, nid): return "NID_%s_UUID" %(nid,) diff --git a/lustre/utils/lmc b/lustre/utils/lmc index 874615f..1d5a281 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -889,7 +889,7 @@ def add_net(gen, lustre, options): if net_type in ('tcp',): port = get_option_int(options, 'port') - elif net_type in ('elan', 'gm', 'openib','iib'): + elif net_type in ('elan', 'gm', 'openib','iib','lo'): port = 0 else: print "Unknown net_type: ", net_type -- 1.8.3.1