From 2cc17c589e498e38ec168651c932fb306555452b Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 18 Nov 2004 01:53:21 +0000 Subject: [PATCH] * Added lbnal --- lustre/configure.in | 2 + lustre/portals/include/linux/kp30.h | 1 + lustre/portals/include/linux/libcfs.h | 1 + lustre/portals/knals/Makefile.in | 1 + lustre/portals/knals/Makefile.mk | 1 + lustre/portals/knals/autoMakefile.am | 2 +- lustre/portals/knals/lbnal/.cvsignore | 10 ++ lustre/portals/knals/lbnal/Makefile.in | 4 + lustre/portals/knals/lbnal/autoMakefile.am | 11 ++ lustre/portals/knals/lbnal/lbnal.c | 170 ++++++++++++++++++ lustre/portals/knals/lbnal/lbnal.h | 77 ++++++++ lustre/portals/knals/lbnal/lbnal_cb.c | 272 +++++++++++++++++++++++++++++ lustre/portals/utils/portals.c | 1 + lustre/ptlrpc/events.c | 1 + lustre/utils/lconf | 5 + lustre/utils/lmc | 2 +- 16 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 lustre/portals/knals/lbnal/.cvsignore create mode 100644 lustre/portals/knals/lbnal/Makefile.in create mode 100644 lustre/portals/knals/lbnal/autoMakefile.am create mode 100644 lustre/portals/knals/lbnal/lbnal.c create mode 100644 lustre/portals/knals/lbnal/lbnal.h create mode 100644 lustre/portals/knals/lbnal/lbnal_cb.c diff --git a/lustre/configure.in b/lustre/configure.in index 479bf65..696e611 100644 --- a/lustre/configure.in +++ b/lustre/configure.in @@ -239,6 +239,8 @@ portals/knals/qswnal/Makefile portals/knals/qswnal/autoMakefile portals/knals/socknal/Makefile portals/knals/socknal/autoMakefile +portals/knals/lbnal/Makefile +portals/knals/lbnal/autoMakefile portals/libcfs/Makefile portals/libcfs/autoMakefile portals/portals/Makefile diff --git a/lustre/portals/include/linux/kp30.h b/lustre/portals/include/linux/kp30.h index 1162c1b..6ddc042 100644 --- a/lustre/portals/include/linux/kp30.h +++ b/lustre/portals/include/linux/kp30.h @@ -651,6 +651,7 @@ enum { ROUTER = 6, OPENIBNAL = 7, IIBNAL = 8, + LBNAL = 9, NAL_ENUM_END_MARKER }; diff --git a/lustre/portals/include/linux/libcfs.h b/lustre/portals/include/linux/libcfs.h index 17f78b7..433344a 100644 --- a/lustre/portals/include/linux/libcfs.h +++ b/lustre/portals/include/linux/libcfs.h @@ -109,6 +109,7 @@ struct ptldebug_header { #define S_SM 0x00800000 #define S_ASOBD 0x01000000 #define S_CONFOBD 0x02000000 +#define S_LBNAL 0x04000000 /* If you change these values, please keep portals/utils/debug.c * up to date! */ diff --git a/lustre/portals/knals/Makefile.in b/lustre/portals/knals/Makefile.in index 9763d14..0650974 100644 --- a/lustre/portals/knals/Makefile.in +++ b/lustre/portals/knals/Makefile.in @@ -3,5 +3,6 @@ @BUILD_IIBNAL_TRUE@subdir-m += iibnal @BUILD_QSWNAL_TRUE@subdir-m += qswnal subdir-m += socknal +subdir-m += lbnal @INCLUDE_RULES@ diff --git a/lustre/portals/knals/Makefile.mk b/lustre/portals/knals/Makefile.mk index cd5d9d6..1d8e79e 100644 --- a/lustre/portals/knals/Makefile.mk +++ b/lustre/portals/knals/Makefile.mk @@ -1,4 +1,5 @@ include $(obj)/../Kernelenv obj-y = socknal/ +obj-y = lbnal/ # more coming... diff --git a/lustre/portals/knals/autoMakefile.am b/lustre/portals/knals/autoMakefile.am index 0090364..37de2d2 100644 --- a/lustre/portals/knals/autoMakefile.am +++ b/lustre/portals/knals/autoMakefile.am @@ -3,4 +3,4 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution -SUBDIRS = gmnal iibnal openibnal qswnal socknal +SUBDIRS = gmnal iibnal openibnal qswnal socknal lbnal diff --git a/lustre/portals/knals/lbnal/.cvsignore b/lustre/portals/knals/lbnal/.cvsignore new file mode 100644 index 0000000..5ed596b --- /dev/null +++ b/lustre/portals/knals/lbnal/.cvsignore @@ -0,0 +1,10 @@ +.deps +Makefile +.*.cmd +autoMakefile.in +autoMakefile +*.ko +*.mod.c +.*.flags +.tmp_versions +.depend diff --git a/lustre/portals/knals/lbnal/Makefile.in b/lustre/portals/knals/lbnal/Makefile.in new file mode 100644 index 0000000..0b4ca19 --- /dev/null +++ b/lustre/portals/knals/lbnal/Makefile.in @@ -0,0 +1,4 @@ +MODULES := klbnal +klbnal-objs := lbnal.o lbnal_cb.o + +@INCLUDE_RULES@ diff --git a/lustre/portals/knals/lbnal/autoMakefile.am b/lustre/portals/knals/lbnal/autoMakefile.am new file mode 100644 index 0000000..103e2df --- /dev/null +++ b/lustre/portals/knals/lbnal/autoMakefile.am @@ -0,0 +1,11 @@ +# Copyright (C) 2001 Cluster File Systems, Inc. +# +# This code is issued under the GNU General Public License. +# See the file COPYING in this distribution + +if MODULES +modulenet_DATA = klbnal$(KMODEXT) +endif + +MOSTLYCLEANFILES = *.o *.ko *.mod.c +DIST_SOURCES = $(klbnal-objs:%.o=%.c) lbnal.h diff --git a/lustre/portals/knals/lbnal/lbnal.c b/lustre/portals/knals/lbnal/lbnal.c new file mode 100644 index 0000000..cfb8614 --- /dev/null +++ b/lustre/portals/knals/lbnal/lbnal.c @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/lustre/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lbnal.h" + +nal_t klbnal_api; +klbnal_data_t klbnal_data; +ptl_handle_ni_t klbnal_ni; + + +int +klbnal_cmd (struct portals_cfg *pcfg, void *private) +{ + LASSERT (pcfg != NULL); + + switch (pcfg->pcfg_command) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID to "LPX64" (was "LPX64")\n", + pcfg->pcfg_nid, klbnal_lib.libnal_ni.ni_pid.nid); + klbnal_lib.libnal_ni.ni_pid.nid = pcfg->pcfg_nid; + return (0); + + default: + return (-EINVAL); + } +} + +static void +klbnal_shutdown(nal_t *nal) +{ + unsigned long flags; + + /* NB The first ref was this module! */ + if (nal->nal_refct != 0) + return; + + CDEBUG (D_NET, "shutdown\n"); + LASSERT (nal == &klbnal_api); + + switch (klbnal_data.klb_init) + { + default: + LASSERT (0); + + case KLB_INIT_ALL: + libcfs_nal_cmd_unregister(LBNAL); + /* fall through */ + + case KLB_INIT_LIB: + lib_fini (&klbnal_lib); + break; + + case KLB_INIT_NOTHING: + return; + } + + memset(&klbnal_data, 0, sizeof (klbnal_data)); + + CDEBUG (D_MALLOC, "done kmem %d\n", atomic_read(&portal_kmemory)); + + printk (KERN_INFO "Lustre: LB NAL unloaded (final mem %d)\n", + atomic_read(&portal_kmemory)); + PORTAL_MODULE_UNUSE; +} + +static int +klbnal_startup (nal_t *nal, ptl_pid_t requested_pid, + ptl_ni_limits_t *requested_limits, + ptl_ni_limits_t *actual_limits) +{ + int rc; + int i; + ptl_process_id_t my_process_id; + int pkmem = atomic_read(&portal_kmemory); + + LASSERT (nal == &klbnal_api); + + if (nal->nal_refct != 0) { + if (actual_limits != NULL) + *actual_limits = klbnal_lib.libnal_ni.ni_actual_limits; + return (PTL_OK); + } + + LASSERT (klbnal_data.klb_init == KLB_INIT_NOTHING); + + CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory)); + + /* ensure all pointers NULL etc */ + memset (&klbnal_data, 0, sizeof (klbnal_data)); + + my_process_id.nid = 0; + my_process_id.pid = requested_pid; + + rc = lib_init(&klbnal_lib, nal, my_process_id, + requested_limits, actual_limits); + if (rc != PTL_OK) { + CERROR ("lib_init failed %d\n", rc); + klbnal_shutdown (nal); + return (rc); + } + + klbnal_data.klb_init = KLB_INIT_LIB; + + rc = libcfs_nal_cmd_register (LBNAL, &klbnal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + klbnal_shutdown (nal); + return (PTL_FAIL); + } + + klbnal_data.klb_init = KLB_INIT_ALL; + + printk(KERN_INFO "Lustre: LB NAL (initial mem %d)\n", pkmem); + PORTAL_MODULE_USE; + + return (PTL_OK); +} + +void __exit +klbnal_finalise (void) +{ + PtlNIFini(klbnal_ni); + + ptl_unregister_nal(LBNAL); +} + +static int __init +klbnal_initialise (void) +{ + int rc; + + klbnal_api.nal_ni_init = klbnal_startup; + klbnal_api.nal_ni_fini = klbnal_shutdown; + + rc = ptl_register_nal(LBNAL, &klbnal_api); + if (rc != PTL_OK) { + CERROR("Can't register LBNAL: %d\n", rc); + return (-ENOMEM); /* or something... */ + } + + return (0); +} + +MODULE_AUTHOR("Cluster File Systems, Inc. "); +MODULE_DESCRIPTION("Loopback NAL v0.01"); +MODULE_LICENSE("GPL"); + +module_init (klbnal_initialise); +module_exit (klbnal_finalise); diff --git a/lustre/portals/knals/lbnal/lbnal.h b/lustre/portals/knals/lbnal/lbnal.h new file mode 100644 index 0000000..f24b409 --- /dev/null +++ b/lustre/portals/knals/lbnal/lbnal.h @@ -0,0 +1,77 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Basic library routines. + * + */ + +#ifndef _LBNAL_H +#define _LBNAL_H +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_SUBSYSTEM S_LBNAL + +#include +#include +#include +#include + +#define KLBD_IOV 153401 +#define KLBD_KIOV 153402 + +typedef struct +{ + unsigned int klbd_type; + unsigned int klbd_niov; + size_t klbd_offset; + size_t klbd_nob; + union { + struct iovec *iov; + ptl_kiov_t *kiov; + } klbd_iov; + +} klb_desc_t; + +typedef struct +{ + char klb_init; /* what's been initialised */ +} klbnal_data_t; + +/* kqn_init state */ +#define KLB_INIT_NOTHING 0 /* MUST BE ZERO so zeroed state is initialised OK */ +#define KLB_INIT_LIB 1 +#define KLB_INIT_ALL 2 + +extern lib_nal_t klbnal_lib; +extern nal_t klbnal_api; +extern klbnal_data_t klbnal_data; + +#endif /* _LBNAL_H */ diff --git a/lustre/portals/knals/lbnal/lbnal_cb.c b/lustre/portals/knals/lbnal/lbnal_cb.c new file mode 100644 index 0000000..1caf476 --- /dev/null +++ b/lustre/portals/knals/lbnal/lbnal_cb.c @@ -0,0 +1,272 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lbnal.h" + +/* + * LIB functions follow + * + */ +static int +klbnal_dist(lib_nal_t *nal, ptl_nid_t nid, unsigned long *dist) +{ + *dist = 0; /* it's me */ + return (0); +} + +static ptl_err_t +klbnal_send (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_offset, + size_t payload_nob) +{ + klb_desc_t klbd = { + .klbd_type = KLBD_IOV, + .klbd_niov = payload_niov, + .klbd_offset = payload_offset, + .klbd_nob = payload_nob, + .klbd_iov.iov = payload_iov}; + ptl_err_t rc; + + LASSERT(nid == klbnal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klbnal_lib, hdr, &klbd); + if (rc == PTL_OK) + lib_finalize(&klbnal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klbnal_send_pages (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_offset, + size_t payload_nob) +{ + klb_desc_t klbd = { + .klbd_type = KLBD_KIOV, + .klbd_niov = payload_niov, + .klbd_offset = payload_offset, + .klbd_nob = payload_nob, + .klbd_iov.kiov = payload_kiov}; + ptl_err_t rc; + + LASSERT(nid == klbnal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klbnal_lib, hdr, &klbd); + if (rc == PTL_OK) + lib_finalize(&klbnal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klbnal_recv(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t mlen, + size_t rlen) +{ + klb_desc_t *klbd = (klb_desc_t *)private; + + /* I only handle mapped->mapped matches */ + LASSERT(klbd->klbd_type == KLBD_IOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= iov->iov_len) { + offset -= iov->iov_len; + iov++; + niov--; + LASSERT(niov > 0); + } + + while (klbd->klbd_offset >= klbd->klbd_iov.iov->iov_len) { + klbd->klbd_offset -= klbd->klbd_iov.iov->iov_len; + klbd->klbd_iov.iov++; + klbd->klbd_niov--; + LASSERT(klbd->klbd_niov > 0); + } + + do { + int fraglen = MIN(iov->iov_len - offset, + klbd->klbd_iov.iov->iov_len - klbd->klbd_offset); + + LASSERT(niov > 0); + LASSERT(klbd->klbd_niov > 0); + + if (fraglen > mlen) + fraglen = mlen; + + memcpy((void *)((unsigned long)iov->iov_base + offset), + (void *)((unsigned long)klbd->klbd_iov.iov->iov_base + + klbd->klbd_offset), + fraglen); + + if (offset + fraglen < iov->iov_len) { + offset += fraglen; + } else { + offset = 0; + iov++; + niov--; + } + + if (klbd->klbd_offset + fraglen < klbd->klbd_iov.iov->iov_len ) { + klbd->klbd_offset += fraglen; + } else { + klbd->klbd_offset = 0; + klbd->klbd_iov.iov++; + klbd->klbd_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + lib_finalize(&klbnal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +static ptl_err_t +klbnal_recv_pages(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + ptl_kiov_t *kiov, + size_t offset, + size_t mlen, + size_t rlen) +{ + void *srcaddr = NULL; + void *dstaddr = NULL; + unsigned long srcfrag = 0; + unsigned long dstfrag = 0; + unsigned long fraglen; + klb_desc_t *klbd = (klb_desc_t *)private; + + /* I only handle unmapped->unmapped matches */ + LASSERT(klbd->klbd_type == KLBD_KIOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= kiov->kiov_len) { + offset -= kiov->kiov_len; + kiov++; + niov--; + LASSERT(niov > 0); + } + + while (klbd->klbd_offset >= klbd->klbd_iov.kiov->kiov_len) { + klbd->klbd_offset -= klbd->klbd_iov.kiov->kiov_len; + klbd->klbd_iov.kiov++; + klbd->klbd_niov--; + LASSERT(klbd->klbd_niov > 0); + } + + do { + /* CAVEAT EMPTOR: I kmap 2 pages at once == slight risk of deadlock */ + LASSERT(niov > 0); + if (dstaddr == NULL) { + dstaddr = (void *)((unsigned long)kmap(kiov->kiov_page) + + kiov->kiov_offset + offset); + dstfrag = kiov->kiov_len - offset; + } + + LASSERT(klbd->klbd_niov > 0); + if (srcaddr == NULL) { + srcaddr = (void *)((unsigned long)kmap(klbd->klbd_iov.kiov->kiov_page) + + klbd->klbd_iov.kiov->kiov_offset + klbd->klbd_offset); + srcfrag = klbd->klbd_iov.kiov->kiov_len - klbd->klbd_offset; + } + + fraglen = MIN(srcfrag, dstfrag); + if (fraglen > mlen) + fraglen = mlen; + + memcpy(dstaddr, srcaddr, fraglen); + + if (fraglen < dstfrag) { + dstfrag -= fraglen; + dstaddr = (void *)((unsigned long)dstaddr + fraglen); + } else { + kunmap(kiov->kiov_page); + dstaddr = NULL; + offset = 0; + kiov++; + niov--; + } + + if (fraglen < srcfrag) { + srcfrag -= fraglen; + srcaddr = (void *)((unsigned long)srcaddr + fraglen); + } else { + kunmap(klbd->klbd_iov.kiov->kiov_page); + srcaddr = NULL; + klbd->klbd_offset = 0; + klbd->klbd_iov.kiov++; + klbd->klbd_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + if (dstaddr != NULL) + kunmap(kiov->kiov_page); + + if (srcaddr != NULL) + kunmap(klbd->klbd_iov.kiov->kiov_page); + + lib_finalize(&klbnal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +lib_nal_t klbnal_lib = +{ + libnal_data: &klbnal_data, /* NAL private data */ + libnal_send: klbnal_send, + libnal_send_pages: klbnal_send_pages, + libnal_recv: klbnal_recv, + libnal_recv_pages: klbnal_recv_pages, + libnal_dist: klbnal_dist +}; diff --git a/lustre/portals/utils/portals.c b/lustre/portals/utils/portals.c index d5d29dc..3f20831 100644 --- a/lustre/portals/utils/portals.c +++ b/lustre/portals/utils/portals.c @@ -71,6 +71,7 @@ static name2num_t nalnames[] = { {"gm", GMNAL}, {"openib", OPENIBNAL}, {"iib", IIBNAL}, + {"lb", LBNAL}, {NULL, -1} }; diff --git a/lustre/ptlrpc/events.c b/lustre/ptlrpc/events.c index b357c77..0ade02c 100644 --- a/lustre/ptlrpc/events.c +++ b/lustre/ptlrpc/events.c @@ -623,6 +623,7 @@ int ptlrpc_init_portals(void) {OPENIBNAL, "openibnal"}, {IIBNAL, "iibnal"}, {TCPNAL, "tcpnal"}, + {LBNAL, "lbnal"}, #else {CRAY_KB_ERNAL, "cray_kb_ernal"}, #endif diff --git a/lustre/utils/lconf b/lustre/utils/lconf index b1cef54..ff4b363 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -114,6 +114,7 @@ subsystem_names = { "ptlrouter" : (1 << 20), "cobd" : (1 << 21), "ibnal" : (1 << 22), + "lbnal" : (1 << 23), } @@ -990,6 +991,8 @@ def sys_get_local_address(net_type, wildcard, cluster_id): local = elan_id except IOError, e: log(e) + elif net_type == 'lb': + fixme("automatic local address for loopback") elif net_type == 'gm': fixme("automatic local address for GM") @@ -1206,6 +1209,8 @@ class Network(Module): self.add_portals_module("knals/openibnal", 'kopenibnal') if self.net_type == 'iib': self.add_portals_module("knals/iibnal", 'kiibnal') + if self.net_type == 'lb': + self.add_portals_module("knals/lbnal", 'klbnal') def nid_to_uuid(self, nid): return "NID_%s_UUID" %(nid,) diff --git a/lustre/utils/lmc b/lustre/utils/lmc index 0d34ace2..9623a41 100755 --- a/lustre/utils/lmc +++ b/lustre/utils/lmc @@ -648,7 +648,7 @@ def add_net(gen, lustre, options): if net_type in ('tcp',): port = get_option_int(options, 'port') - elif net_type in ('elan', 'gm', 'openib','iib'): + elif net_type in ('elan', 'gm', 'openib','iib','lb'): port = 0 else: print "Unknown net_type: ", net_type -- 1.8.3.1