From 3ecd3370cf7e8facdea2e53a2fc222cbaa65273f Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 18 Nov 2004 11:19:57 +0000 Subject: [PATCH] * Added lonal (loopback NAL) --- lnet/klnds/lolnd/.cvsignore | 10 ++ lnet/klnds/lolnd/Makefile.in | 4 + lnet/klnds/lolnd/autoMakefile.am | 11 ++ lnet/klnds/lolnd/lolnd.c | 170 ++++++++++++++++++++++++ lnet/klnds/lolnd/lolnd.h | 77 +++++++++++ lnet/klnds/lolnd/lolnd_cb.c | 272 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 544 insertions(+) create mode 100644 lnet/klnds/lolnd/.cvsignore create mode 100644 lnet/klnds/lolnd/Makefile.in create mode 100644 lnet/klnds/lolnd/autoMakefile.am create mode 100644 lnet/klnds/lolnd/lolnd.c create mode 100644 lnet/klnds/lolnd/lolnd.h create mode 100644 lnet/klnds/lolnd/lolnd_cb.c diff --git a/lnet/klnds/lolnd/.cvsignore b/lnet/klnds/lolnd/.cvsignore new file mode 100644 index 0000000..5ed596b --- /dev/null +++ b/lnet/klnds/lolnd/.cvsignore @@ -0,0 +1,10 @@ +.deps +Makefile +.*.cmd +autoMakefile.in +autoMakefile +*.ko +*.mod.c +.*.flags +.tmp_versions +.depend diff --git a/lnet/klnds/lolnd/Makefile.in b/lnet/klnds/lolnd/Makefile.in new file mode 100644 index 0000000..222e861 --- /dev/null +++ b/lnet/klnds/lolnd/Makefile.in @@ -0,0 +1,4 @@ +MODULES := klonal +klonal-objs := lonal.o lonal_cb.o + +@INCLUDE_RULES@ diff --git a/lnet/klnds/lolnd/autoMakefile.am b/lnet/klnds/lolnd/autoMakefile.am new file mode 100644 index 0000000..d1ef995 --- /dev/null +++ b/lnet/klnds/lolnd/autoMakefile.am @@ -0,0 +1,11 @@ +# Copyright (C) 2001 Cluster File Systems, Inc. +# +# This code is issued under the GNU General Public License. +# See the file COPYING in this distribution + +if MODULES +modulenet_DATA = klonal$(KMODEXT) +endif + +MOSTLYCLEANFILES = *.o *.ko *.mod.c +DIST_SOURCES = $(klonal-objs:%.o=%.c) lonal.h diff --git a/lnet/klnds/lolnd/lolnd.c b/lnet/klnds/lolnd/lolnd.c new file mode 100644 index 0000000..055e002 --- /dev/null +++ b/lnet/klnds/lolnd/lolnd.c @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/lustre/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +nal_t klonal_api; +klonal_data_t klonal_data; +ptl_handle_ni_t klonal_ni; + + +int +klonal_cmd (struct portals_cfg *pcfg, void *private) +{ + LASSERT (pcfg != NULL); + + switch (pcfg->pcfg_command) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID to "LPX64" (was "LPX64")\n", + pcfg->pcfg_nid, klonal_lib.libnal_ni.ni_pid.nid); + klonal_lib.libnal_ni.ni_pid.nid = pcfg->pcfg_nid; + return (0); + + default: + return (-EINVAL); + } +} + +static void +klonal_shutdown(nal_t *nal) +{ + unsigned long flags; + + /* NB The first ref was this module! */ + if (nal->nal_refct != 0) + return; + + CDEBUG (D_NET, "shutdown\n"); + LASSERT (nal == &klonal_api); + + switch (klonal_data.klo_init) + { + default: + LASSERT (0); + + case KLO_INIT_ALL: + libcfs_nal_cmd_unregister(LONAL); + /* fall through */ + + case KLO_INIT_LIB: + lib_fini (&klonal_lib); + break; + + case KLO_INIT_NOTHING: + return; + } + + memset(&klonal_data, 0, sizeof (klonal_data)); + + CDEBUG (D_MALLOC, "done kmem %d\n", atomic_read(&portal_kmemory)); + + printk (KERN_INFO "Lustre: LO NAL unloaded (final mem %d)\n", + atomic_read(&portal_kmemory)); + PORTAL_MODULE_UNUSE; +} + +static int +klonal_startup (nal_t *nal, ptl_pid_t requested_pid, + ptl_ni_limits_t *requested_limits, + ptl_ni_limits_t *actual_limits) +{ + int rc; + int i; + ptl_process_id_t my_process_id; + int pkmem = atomic_read(&portal_kmemory); + + LASSERT (nal == &klonal_api); + + if (nal->nal_refct != 0) { + if (actual_limits != NULL) + *actual_limits = klonal_lib.libnal_ni.ni_actual_limits; + return (PTL_OK); + } + + LASSERT (klonal_data.klo_init == KLO_INIT_NOTHING); + + CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory)); + + /* ensure all pointers NULL etc */ + memset (&klonal_data, 0, sizeof (klonal_data)); + + my_process_id.nid = 0; + my_process_id.pid = requested_pid; + + rc = lib_init(&klonal_lib, nal, my_process_id, + requested_limits, actual_limits); + if (rc != PTL_OK) { + CERROR ("lib_init failed %d\n", rc); + klonal_shutdown (nal); + return (rc); + } + + klonal_data.klo_init = KLO_INIT_LIB; + + rc = libcfs_nal_cmd_register (LONAL, &klonal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + klonal_shutdown (nal); + return (PTL_FAIL); + } + + klonal_data.klo_init = KLO_INIT_ALL; + + printk(KERN_INFO "Lustre: LO NAL (initial mem %d)\n", pkmem); + PORTAL_MODULE_USE; + + return (PTL_OK); +} + +void __exit +klonal_finalise (void) +{ + PtlNIFini(klonal_ni); + + ptl_unregister_nal(LONAL); +} + +static int __init +klonal_initialise (void) +{ + int rc; + + klonal_api.nal_ni_init = klonal_startup; + klonal_api.nal_ni_fini = klonal_shutdown; + + rc = ptl_register_nal(LONAL, &klonal_api); + if (rc != PTL_OK) { + CERROR("Can't register LONAL: %d\n", rc); + return (-ENOMEM); /* or something... */ + } + + return (0); +} + +MODULE_AUTHOR("Cluster File Systems, Inc. "); +MODULE_DESCRIPTION("Loopback NAL v0.01"); +MODULE_LICENSE("GPL"); + +module_init (klonal_initialise); +module_exit (klonal_finalise); diff --git a/lnet/klnds/lolnd/lolnd.h b/lnet/klnds/lolnd/lolnd.h new file mode 100644 index 0000000..f1675a7 --- /dev/null +++ b/lnet/klnds/lolnd/lolnd.h @@ -0,0 +1,77 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Basic library routines. + * + */ + +#ifndef _LONAL_H +#define _LONAL_H +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_SUBSYSTEM S_LONAL + +#include +#include +#include +#include + +#define KLOD_IOV 153401 +#define KLOD_KIOV 153402 + +typedef struct +{ + unsigned int klod_type; + unsigned int klod_niov; + size_t klod_offset; + size_t klod_nob; + union { + struct iovec *iov; + ptl_kiov_t *kiov; + } klod_iov; + +} klo_desc_t; + +typedef struct +{ + char klo_init; /* what's been initialised */ +} klonal_data_t; + +/* kqn_init state */ +#define KLO_INIT_NOTHING 0 /* MUST BE ZERO so zeroed state is initialised OK */ +#define KLO_INIT_LIB 1 +#define KLO_INIT_ALL 2 + +extern lib_nal_t klonal_lib; +extern nal_t klonal_api; +extern klonal_data_t klonal_data; + +#endif /* _LONAL_H */ diff --git a/lnet/klnds/lolnd/lolnd_cb.c b/lnet/klnds/lolnd/lolnd_cb.c new file mode 100644 index 0000000..dccc81a --- /dev/null +++ b/lnet/klnds/lolnd/lolnd_cb.c @@ -0,0 +1,272 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Copyright (C) 2002, Lawrence Livermore National Labs (LLNL) + * W. Marcus Miller - Based on ksocknal + * + * This file is part of Portals, http://www.sf.net/projects/sandiaportals/ + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "lonal.h" + +/* + * LIB functions follow + * + */ +static int +klonal_dist(lib_nal_t *nal, ptl_nid_t nid, unsigned long *dist) +{ + *dist = 0; /* it's me */ + return (0); +} + +static ptl_err_t +klonal_send (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_IOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.iov = payload_iov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_send_pages (lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + ptl_hdr_t *hdr, + int type, + ptl_nid_t nid, + ptl_pid_t pid, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_offset, + size_t payload_nob) +{ + klo_desc_t klod = { + .klod_type = KLOD_KIOV, + .klod_niov = payload_niov, + .klod_offset = payload_offset, + .klod_nob = payload_nob, + .klod_iov.kiov = payload_kiov}; + ptl_err_t rc; + + LASSERT(nid == klonal_lib.libnal_ni.ni_pid.nid); + + rc = lib_parse(&klonal_lib, hdr, &klod); + if (rc == PTL_OK) + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + + return rc; +} + +static ptl_err_t +klonal_recv(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + struct iovec *iov, + size_t offset, + size_t mlen, + size_t rlen) +{ + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle mapped->mapped matches */ + LASSERT(klod->klod_type == KLOD_IOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= iov->iov_len) { + offset -= iov->iov_len; + iov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.iov->iov_len) { + klod->klod_offset -= klod->klod_iov.iov->iov_len; + klod->klod_iov.iov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + int fraglen = MIN(iov->iov_len - offset, + klod->klod_iov.iov->iov_len - klod->klod_offset); + + LASSERT(niov > 0); + LASSERT(klod->klod_niov > 0); + + if (fraglen > mlen) + fraglen = mlen; + + memcpy((void *)((unsigned long)iov->iov_base + offset), + (void *)((unsigned long)klod->klod_iov.iov->iov_base + + klod->klod_offset), + fraglen); + + if (offset + fraglen < iov->iov_len) { + offset += fraglen; + } else { + offset = 0; + iov++; + niov--; + } + + if (klod->klod_offset + fraglen < klod->klod_iov.iov->iov_len ) { + klod->klod_offset += fraglen; + } else { + klod->klod_offset = 0; + klod->klod_iov.iov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +static ptl_err_t +klonal_recv_pages(lib_nal_t *nal, + void *private, + lib_msg_t *libmsg, + unsigned int niov, + ptl_kiov_t *kiov, + size_t offset, + size_t mlen, + size_t rlen) +{ + void *srcaddr = NULL; + void *dstaddr = NULL; + unsigned long srcfrag = 0; + unsigned long dstfrag = 0; + unsigned long fraglen; + klo_desc_t *klod = (klo_desc_t *)private; + + /* I only handle unmapped->unmapped matches */ + LASSERT(klod->klod_type == KLOD_KIOV); + + if (mlen == 0) + return PTL_OK; + + while (offset >= kiov->kiov_len) { + offset -= kiov->kiov_len; + kiov++; + niov--; + LASSERT(niov > 0); + } + + while (klod->klod_offset >= klod->klod_iov.kiov->kiov_len) { + klod->klod_offset -= klod->klod_iov.kiov->kiov_len; + klod->klod_iov.kiov++; + klod->klod_niov--; + LASSERT(klod->klod_niov > 0); + } + + do { + /* CAVEAT EMPTOR: I kmap 2 pages at once == slight risk of deadlock */ + LASSERT(niov > 0); + if (dstaddr == NULL) { + dstaddr = (void *)((unsigned long)kmap(kiov->kiov_page) + + kiov->kiov_offset + offset); + dstfrag = kiov->kiov_len - offset; + } + + LASSERT(klod->klod_niov > 0); + if (srcaddr == NULL) { + srcaddr = (void *)((unsigned long)kmap(klod->klod_iov.kiov->kiov_page) + + klod->klod_iov.kiov->kiov_offset + klod->klod_offset); + srcfrag = klod->klod_iov.kiov->kiov_len - klod->klod_offset; + } + + fraglen = MIN(srcfrag, dstfrag); + if (fraglen > mlen) + fraglen = mlen; + + memcpy(dstaddr, srcaddr, fraglen); + + if (fraglen < dstfrag) { + dstfrag -= fraglen; + dstaddr = (void *)((unsigned long)dstaddr + fraglen); + } else { + kunmap(kiov->kiov_page); + dstaddr = NULL; + offset = 0; + kiov++; + niov--; + } + + if (fraglen < srcfrag) { + srcfrag -= fraglen; + srcaddr = (void *)((unsigned long)srcaddr + fraglen); + } else { + kunmap(klod->klod_iov.kiov->kiov_page); + srcaddr = NULL; + klod->klod_offset = 0; + klod->klod_iov.kiov++; + klod->klod_niov--; + } + + mlen -= fraglen; + } while (mlen > 0); + + if (dstaddr != NULL) + kunmap(kiov->kiov_page); + + if (srcaddr != NULL) + kunmap(klod->klod_iov.kiov->kiov_page); + + lib_finalize(&klonal_lib, private, libmsg, PTL_OK); + return PTL_OK; +} + +lib_nal_t klonal_lib = +{ + libnal_data: &klonal_data, /* NAL private data */ + libnal_send: klonal_send, + libnal_send_pages: klonal_send_pages, + libnal_recv: klonal_recv, + libnal_recv_pages: klonal_recv_pages, + libnal_dist: klonal_dist +}; -- 1.8.3.1