From: Isaac Huang Date: Tue, 6 Jul 2010 17:24:13 +0000 (-0600) Subject: b=22745 Removed gmlnd from lnet. X-Git-Tag: 2.0.0.51~37 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=c660aac3f17e4100b4bddb8985b61c1f8d96ecb0 b=22745 Removed gmlnd from lnet. Removed obsolete gmlnd from lnet. i=maxim --- diff --git a/libcfs/include/libcfs/libcfs_ioctl.h b/libcfs/include/libcfs/libcfs_ioctl.h index 31d5a0a..3a27552 100644 --- a/libcfs/include/libcfs/libcfs_ioctl.h +++ b/libcfs/include/libcfs/libcfs_ioctl.h @@ -149,9 +149,8 @@ struct libcfs_ioctl_handler { #define IOC_LIBCFS_ADD_INTERFACE _IOWR('e', 78, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_DEL_INTERFACE _IOWR('e', 79, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_GET_INTERFACE _IOWR('e', 80, IOCTL_LIBCFS_TYPE) -#define IOC_LIBCFS_GET_GMID _IOWR('e', 81, IOCTL_LIBCFS_TYPE) -#define IOC_LIBCFS_MAX_NR 81 +#define IOC_LIBCFS_MAX_NR 80 static inline int libcfs_ioctl_packlen(struct libcfs_ioctl_data *data) { diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h index 02cd0f8..eb86acb 100644 --- a/libcfs/include/libcfs/libcfs_private.h +++ b/libcfs/include/libcfs/libcfs_private.h @@ -427,7 +427,7 @@ enum { * network addresses depend on them... */ QSWLND = 1, SOCKLND = 2, - GMLND = 3, + GMLND = 3, /* obsolete, keep it so that libcfs_nid2str works */ PTLLND = 4, O2IBLND = 5, CIBLND = 6, diff --git a/libcfs/libcfs/nidstrings.c b/libcfs/libcfs/nidstrings.c index 15c0a67..d5b03ed 100644 --- a/libcfs/libcfs/nidstrings.c +++ b/libcfs/libcfs/nidstrings.c @@ -576,7 +576,7 @@ libcfs_str2anynid(lnet_nid_t *nidp, const char *str) * '-' '/' * :== | * :== "lo" | "tcp" | "o2ib" | "cib" | "openib" | "iib" | - * "vib" | "ra" | "elan" | "gm" | "mx" | "ptl" + * "vib" | "ra" | "elan" | "mx" | "ptl" * \endverbatim */ diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index ea5a833..70ce4a7 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -255,106 +255,6 @@ AC_SUBST(QSWCPPFLAGS) AC_SUBST(QSWLND) ]) -# -# LN_CONFIG_GM -# -# check if GM support is available -# -AC_DEFUN([LN_CONFIG_GM],[ -AC_MSG_CHECKING([whether to enable GM support]) -AC_ARG_WITH([gm], - AC_HELP_STRING([--with-gm=path-to-gm-source-tree], - [build gmlnd against path]), - [ - case $with_gm in - no) ENABLE_GM=0 - ;; - *) ENABLE_GM=1 - GM_SRC="$with_gm" - ;; - esac - ],[ - ENABLE_GM=0 - ]) -AC_ARG_WITH([gm-install], - AC_HELP_STRING([--with-gm-install=path-to-gm-install-tree], - [say where GM has been installed]), - [ - GM_INSTALL=$with_gm_install - ],[ - GM_INSTALL="/opt/gm" - ]) -if test $ENABLE_GM -eq 0; then - AC_MSG_RESULT([no]) -else - AC_MSG_RESULT([yes]) - - GMLND="gmlnd" - GMCPPFLAGS="-I$GM_SRC/include -I$GM_SRC/drivers -I$GM_SRC/drivers/linux/gm" - - if test -f $GM_INSTALL/lib/libgm.a -o \ - -f $GM_INSTALL/lib64/libgm.a; then - GMLIBS="-L$GM_INSTALL/lib -L$GM_INSTALL/lib64" - else - AC_MSG_ERROR([Cant find GM libraries under $GM_INSTALL]) - fi - - EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" - EXTRA_KCFLAGS="$GMCPPFLAGS -DGM_KERNEL $EXTRA_KCFLAGS" - - AC_MSG_CHECKING([that code using GM compiles with given path]) - LB_LINUX_TRY_COMPILE([ - #define GM_STRONG_TYPES 1 - #ifdef VERSION - #undef VERSION - #endif - #include "gm.h" - #include "gm_internal.h" - ],[ - struct gm_port *port = NULL; - gm_recv_event_t *rxevent = gm_blocking_receive_no_spin(port); - return 0; - ],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_ERROR([Bad --with-gm path]) - ]) - - AC_MSG_CHECKING([that GM has gm_register_memory_ex_phys()]) - LB_LINUX_TRY_COMPILE([ - #define GM_STRONG_TYPES 1 - #ifdef VERSION - #undef VERSION - #endif - #include "gm.h" - #include "gm_internal.h" - ],[ - gm_status_t gmrc; - struct gm_port *port = NULL; - gm_u64_t phys = 0; - gm_up_t pvma = 0; - - gmrc = gm_register_memory_ex_phys(port, phys, 100, pvma); - return 0; - ],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no. -Please patch the GM sources as follows... - cd $GM_SRC - patch -p0 < $PWD/lnet/klnds/gmlnd/gm-reg-phys.patch -...then rebuild and re-install them]) - AC_MSG_ERROR([Can't build GM without gm_register_memory_ex_phys()]) - ]) - - EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" -fi -AC_SUBST(GMCPPFLAGS) -AC_SUBST(GMLIBS) -AC_SUBST(GMLND) -]) - # # LN_CONFIG_MX @@ -949,7 +849,6 @@ AC_DEFUN([LN_PROG_LINUX], LN_CONFIG_AFFINITY LN_CONFIG_BACKOFF LN_CONFIG_QUADRICS -LN_CONFIG_GM LN_CONFIG_OPENIB LN_CONFIG_CIB LN_CONFIG_VIB @@ -1098,7 +997,6 @@ LN_CONFIG_USOCKLND # AC_DEFUN([LN_CONDITIONALS], [AM_CONDITIONAL(BUILD_QSWLND, test x$QSWLND = "xqswlnd") -AM_CONDITIONAL(BUILD_GMLND, test x$GMLND = "xgmlnd") AM_CONDITIONAL(BUILD_MXLND, test x$MXLND = "xmxlnd") AM_CONDITIONAL(BUILD_O2IBLND, test x$O2IBLND = "xo2iblnd") AM_CONDITIONAL(BUILD_OPENIBLND, test x$OPENIBLND = "xopeniblnd") @@ -1128,10 +1026,8 @@ lnet/include/lnet/Makefile lnet/include/lnet/linux/Makefile lnet/klnds/Makefile lnet/klnds/autoMakefile -lnet/klnds/gmlnd/Makefile lnet/klnds/mxlnd/autoMakefile lnet/klnds/mxlnd/Makefile -lnet/klnds/gmlnd/autoMakefile lnet/klnds/openiblnd/Makefile lnet/klnds/openiblnd/autoMakefile lnet/klnds/o2iblnd/Makefile diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 63d497a..4312501 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -160,7 +160,6 @@ typedef struct { #define LNET_PROTO_QSW_MAGIC 0x0be91b93 #define LNET_PROTO_TCP_MAGIC 0xeebc0ded #define LNET_PROTO_PTL_MAGIC 0x50746C4E /* 'PtlN' unique magic */ -#define LNET_PROTO_GM_MAGIC 0x6d797269 /* 'myri'! */ #define LNET_PROTO_MX_MAGIC 0x4d583130 /* 'MX10'! */ #define LNET_PROTO_ACCEPTOR_MAGIC 0xacce7100 #define LNET_PROTO_PING_MAGIC 0x70696E67 /* 'ping' */ diff --git a/lnet/include/lnet/lnet-sysctl.h b/lnet/include/lnet/lnet-sysctl.h index de2506d..dfa5dc4 100644 --- a/lnet/include/lnet/lnet-sysctl.h +++ b/lnet/include/lnet/lnet-sysctl.h @@ -42,7 +42,6 @@ #ifndef HAVE_SYSCTL_UNNUMBERED #define CTL_KRANAL 201 -#define CTL_GMLND 202 #define CTL_KIBNAL 203 #define CTL_IIBBLND 204 #define CTL_O2IBLND 205 @@ -55,7 +54,6 @@ #else #define CTL_KRANAL CTL_UNNUMBERED -#define CTL_GMLND CTL_UNNUMBERED #define CTL_KIBNAL CTL_UNNUMBERED #define CTL_IIBLND CTL_UNNUMBERED #define CTL_O2IBLND CTL_UNNUMBERED diff --git a/lnet/klnds/Makefile.in b/lnet/klnds/Makefile.in index d4e034c..0e8bb60 100644 --- a/lnet/klnds/Makefile.in +++ b/lnet/klnds/Makefile.in @@ -1,4 +1,3 @@ -@BUILD_GMLND_TRUE@subdir-m += gmlnd @BUILD_MXLND_TRUE@subdir-m += mxlnd @BUILD_RALND_TRUE@subdir-m += ralnd @BUILD_O2IBLND_TRUE@subdir-m += o2iblnd diff --git a/lnet/klnds/autoMakefile.am b/lnet/klnds/autoMakefile.am index f70886e..cdd1820 100644 --- a/lnet/klnds/autoMakefile.am +++ b/lnet/klnds/autoMakefile.am @@ -34,4 +34,4 @@ # Lustre is a trademark of Sun Microsystems, Inc. # -SUBDIRS = socklnd qswlnd gmlnd mxlnd openiblnd iiblnd viblnd ralnd ptllnd ciblnd o2iblnd +SUBDIRS = socklnd qswlnd mxlnd openiblnd iiblnd viblnd ralnd ptllnd ciblnd o2iblnd diff --git a/lnet/klnds/gmlnd/.gitignore b/lnet/klnds/gmlnd/.gitignore deleted file mode 100644 index 803b5fc..0000000 --- a/lnet/klnds/gmlnd/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -/.deps -/Makefile -/autoMakefile.in -/autoMakefile -/*.ko -/*.mod.c -/.*.cmd -/.*.flags -/.tmp_versions -/.depend diff --git a/lnet/klnds/gmlnd/Makefile.in b/lnet/klnds/gmlnd/Makefile.in deleted file mode 100644 index 1aec50d..0000000 --- a/lnet/klnds/gmlnd/Makefile.in +++ /dev/null @@ -1,6 +0,0 @@ -MODULES := kgmlnd -kgmlnd-objs := gmlnd_api.o gmlnd_cb.o gmlnd_comm.o gmlnd_utils.o gmlnd_module.o - -EXTRA_PRE_CFLAGS := @GMCPPFLAGS@ -DGM_KERNEL - -@INCLUDE_RULES@ diff --git a/lnet/klnds/gmlnd/README b/lnet/klnds/gmlnd/README deleted file mode 100644 index ac2e23d..0000000 --- a/lnet/klnds/gmlnd/README +++ /dev/null @@ -1,73 +0,0 @@ -1. This version of the GM nal requires an unreleased extension to the GM API to - map physical memory: gm_register_memory_ex_phys(). This allows it to avoid - ENOMEM problems associated with large contiguous buffer allocation. - -2. ./configure --with-gm= \ - [--with-gm-install=] - - If the sources do not support gm_register_memory_ex_phys(), configure flags - an error. In this case you should apply the patch and rebuild and re-install - GM as directed in the error message. - - By default GM is installed in /opt/gm. If an alternate path was specified to - /binary/GM_INSTALL, you should also specify --with-gm-install - with the same path. - -3. The GM timeout is 300 seconds; i.e. the network may not release resources - claimed by communications stalled with a crashing node for this time. - Default gmnal buffer tuning parameters (see (4) below) have been chosen to - minimize this problem and prevent lustre having to block for resources. - However in some situations, where all network buffers are busy, the default - lustre timeout (various, scaled from the base timeout of 100 seconds) may be - too small and the only solution may be to increase the lustre timeout - dramatically. - -4. The gmnal has the following module parameters... - - gmnal_port The GM port that the NAL will use (default 4) - Change this if it conflicts with site usage. - - gmnal_ntx The number of "normal" transmit descriptors (default - 32). When this pool is exhausted, threads sending - and receiving on the network block until in-progress - transmits have completed. Each descriptor consumes 1 - GM_MTU sized buffer. - - gmnal_ntx_nblk The number of "reserved" transmit descriptors - (default 256). This pool is reserved for responses to - incoming communications that may not block. Increase - only if console error messages indicates the pool - has been exhausted (LustreError: Can't get tx for - msg type...) Each descriptor consumes 1 GM_MTU sized - buffer. - - gmnal_nlarge_tx_bufs The number of 1MByte transmit buffers to reserve at - startup (default 32). This controls the number of - concurrent sends larger that GM_MTU. It can be - reduced to conserve memory, or increased to increase - large message sending concurrency. - - gmnal_nrx_small The number of GM_MTU sized receive buffers posted to - receive from the network (default 128). Increase if - congestion is suspected, however note that the total - number of receives that can be posted at any time is - limited by the number of GM receive tokens - available. If there are too few, this, and - gmnal_nrx_large are scaled back accordingly. - - gmnal_nrx_large The number of 1MByte receive buffers posted to - receive from the network (default 64). Increase if - the number of OST threads is increased. But note - that the total number of receives that can be posted - at any time is limited by the number of GM receive - tokens available. If there are too few, this, and - gmnal_nrx_small are scaled back accordingly. - -5. Network configuration for GM is done in an lmc script as follows... - - GM2NID=${path-to-lustre-tree}/portals/utils/gmnalnid - - ${LMC} --node some_server --add net --nettype gm --nid `$GM2NID -n some_server` - - ${LMC} --node client --add net --nettype gm --nid '*' - diff --git a/lnet/klnds/gmlnd/autoMakefile.am b/lnet/klnds/gmlnd/autoMakefile.am deleted file mode 100644 index 9256988..0000000 --- a/lnet/klnds/gmlnd/autoMakefile.am +++ /dev/null @@ -1,44 +0,0 @@ -# -# GPL HEADER START -# -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 only, -# as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License version 2 for more details (a copy is included -# in the LICENSE file that accompanied this code). -# -# You should have received a copy of the GNU General Public License -# version 2 along with this program; If not, see -# http://www.sun.com/software/products/lustre/docs/GPLv2.pdf -# -# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, -# CA 95054 USA or visit www.sun.com if you need additional information or -# have any questions. -# -# GPL HEADER END -# - -# -# Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. -# Use is subject to license terms. -# - -# -# This file is part of Lustre, http://www.lustre.org/ -# Lustre is a trademark of Sun Microsystems, Inc. -# - -if MODULES -if BUILD_GMLND -modulenet_DATA = kgmlnd$(KMODEXT) -endif -endif - -MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ -DIST_SOURCES = $(kgmlnd-objs:%.o=%.c) gmlnd.h diff --git a/lnet/klnds/gmlnd/gm-reg-phys.patch b/lnet/klnds/gmlnd/gm-reg-phys.patch deleted file mode 100644 index df32a21..0000000 --- a/lnet/klnds/gmlnd/gm-reg-phys.patch +++ /dev/null @@ -1,107 +0,0 @@ -Index: libgm/gm_register.c -=================================================================== -RCS file: /repository/gm/libgm/gm_register.c,v -retrieving revision 1.9.16.3 -diff -u -r1.9.16.3 gm_register.c ---- libgm/gm_register.c 9 Aug 2005 14:37:02 -0000 1.9.16.3 -+++ libgm/gm_register.c 25 Aug 2005 21:35:58 -0000 -@@ -77,20 +77,14 @@ - - */ - --GM_ENTRY_POINT --gm_status_t --gm_register_memory_ex (gm_port_t *p, void *_ptr, gm_size_t length, void *_pvma) -+static gm_status_t -+_gm_register_memory (gm_port_t *p, int is_physical, gm_u64_t ptr, gm_size_t length, gm_up_t pvma) - { - gm_status_t status; -- gm_up_t ptr; -- gm_up_t pvma; - - GM_CALLED_WITH_ARGS (("%p,%p,"GM_U64_TMPL",%p", - p, _ptr, GM_U64_ARG (length), _pvma)); - -- ptr = GM_PTR_TO_UP (_ptr); -- pvma = GM_PTR_TO_UP (_pvma); -- - #if !GM_KERNEL && !GM_CAN_REGISTER_MEMORY - GM_PARAMETER_MAY_BE_UNUSED (p); - GM_PARAMETER_MAY_BE_UNUSED (ptr); -@@ -160,7 +154,7 @@ - status = gm_add_mapping_to_page_table (ps, - ptr + offset, - pvma + offset, -- GM_INVALID_DMA_PAGE); -+ is_physical ? ptr + offset : GM_INVALID_DMA_PAGE); - if (status != GM_SUCCESS) - { - status = GM_INVALID_PARAMETER; -@@ -317,13 +311,31 @@ - - */ - -+#if GM_KERNEL && (GM_CPU_x86 || GM_CPU_x86_64 || GM_CPU_ia64) -+/* only architecture where pci bus addr == physical address can use -+ such a simple scheme */ -+GM_ENTRY_POINT gm_status_t -+gm_register_memory_ex_phys (struct gm_port *p, -+ gm_u64_t phys, gm_size_t length, -+ gm_up_t pvma) -+{ -+ return _gm_register_memory(p, 1, phys, length, (gm_size_t)pvma); -+} -+#endif -+ -+GM_ENTRY_POINT gm_status_t -+gm_register_memory_ex (gm_port_t *p, void *ptr, gm_size_t length, void *pvma) -+{ -+ return _gm_register_memory(p, 0, (gm_size_t)ptr, length, (gm_size_t)pvma); -+} -+ - GM_ENTRY_POINT gm_status_t - gm_register_memory (gm_port_t *p, void *ptr, gm_size_t length) - { - gm_status_t status; - - GM_CALLED_WITH_ARGS (("%p,%p,"GM_U64_TMPL, p, ptr, GM_U64_ARG (length))); -- status = gm_register_memory_ex (p, ptr, length, ptr); -+ status = _gm_register_memory(p, 0, (gm_size_t)ptr, length, (gm_size_t)ptr); - GM_RETURN_STATUS (status); - } - -Index: include/gm.h -=================================================================== -RCS file: /repository/gm/include/gm.h,v -retrieving revision 1.25.10.11 -diff -u -r1.25.10.11 gm.h ---- include/gm.h 14 Mar 2005 21:42:41 -0000 1.25.10.11 -+++ include/gm.h 25 Aug 2005 21:35:58 -0000 -@@ -2676,6 +2676,10 @@ - GM_ENTRY_POINT gm_status_t gm_register_memory_ex (struct gm_port *p, - void *ptr, gm_size_t length, - void *pvma); -+ -+GM_ENTRY_POINT gm_status_t gm_register_memory_ex_phys (struct gm_port *p, -+ gm_u64_t phys, gm_size_t length, -+ gm_up_t pvma); - #endif /* GM_API_VERSION >= GM_API_VERSION_2_0_6 */ - - #if GM_API_VERSION >= GM_API_VERSION_2_1_0 -Index: libgm/gm_reference_api.c -=================================================================== -RCS file: /repository/gm/libgm/gm_reference_api.c,v -retrieving revision 1.3.14.1 -diff -u -r1.3.14.1 gm_reference_api.c ---- libgm/gm_reference_api.c 23 Apr 2004 20:27:29 -0000 1.3.14.1 -+++ libgm/gm_reference_api.c 25 Aug 2005 22:39:20 -0000 -@@ -154,6 +154,9 @@ - GM_REF (gm_register_buffer); - GM_REF (gm_register_memory); - GM_REF (gm_register_memory_ex); -+#if GM_KERNEL && (GM_CPU_x86 || GM_CPU_x86_64 || GM_CPU_ia64) -+GM_REF (gm_register_memory_ex_phys); -+#endif - GM_REF (gm_resume_sending); - GM_REF (gm_send); - GM_REF (gm_send_to_peer); diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h deleted file mode 100644 index 1446644..0000000 --- a/lnet/klnds/gmlnd/gmlnd.h +++ /dev/null @@ -1,265 +0,0 @@ -/* - * -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -/* - * Portals GM kernel NAL header file - * This file makes all declaration and prototypes - * for the API side and CB side of the NAL - */ -#ifndef __INCLUDE_GMNAL_H__ -#define __INCLUDE_GMNAL_H__ - -/* XXX Lustre as of V1.2.2 drop defines VERSION, which causes problems - * when including /include/gm_lanai.h which defines a structure field - * with the name VERSION XXX */ -#ifdef VERSION -# undef VERSION -#endif - -#ifndef EXPORT_SYMTAB -# define EXPORT_SYMTAB -#endif -#ifndef AUTOCONF_INCLUDED -#include -#endif -#include "linux/module.h" -#include "linux/tty.h" -#include "linux/kernel.h" -#include "linux/mm.h" -#include "linux/string.h" -#include "linux/stat.h" -#include "linux/errno.h" -#include "linux/version.h" -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) -#include "linux/buffer_head.h" -#include "linux/fs.h" -#else -#include "linux/locks.h" -#endif -#include "linux/unistd.h" -#include "linux/init.h" -#include "linux/sem.h" -#include "linux/vmalloc.h" -#include "linux/sysctl.h" - -#define DEBUG_SUBSYSTEM S_LND - -#include "libcfs/libcfs.h" -#include "lnet/lnet.h" -#include "lnet/lib-lnet.h" -#include "lnet/lnet-sysctl.h" - -/* undefine these before including the GM headers which clash */ -#undef PACKAGE_BUGREPORT -#undef PACKAGE_NAME -#undef PACKAGE_STRING -#undef PACKAGE_TARNAME -#undef PACKAGE_VERSION - -#define GM_STRONG_TYPES 1 -#ifdef VERSION -#undef VERSION -#endif -#include "gm.h" -#include "gm_internal.h" - -/* Fixed tunables */ -#define GMNAL_RESCHED 100 /* # busy loops to force scheduler to yield */ -#define GMNAL_NETADDR_BASE 0x10000000 /* where we start in network VM */ -#define GMNAL_LARGE_PRIORITY GM_LOW_PRIORITY /* large message GM priority */ -#define GMNAL_SMALL_PRIORITY GM_LOW_PRIORITY /* small message GM priority */ - -/* Wire protocol */ -typedef struct { - lnet_hdr_t gmim_hdr; /* portals header */ - char gmim_payload[0]; /* payload */ -} gmnal_immediate_msg_t; - -typedef struct { - /* First 2 fields fixed FOR ALL TIME */ - __u32 gmm_magic; /* I'm a GM message */ - __u16 gmm_version; /* this is my version number */ - - __u16 gmm_type; /* msg type */ - __u64 gmm_srcnid; /* sender's NID */ - __u64 gmm_dstnid; /* destination's NID */ - union { - gmnal_immediate_msg_t immediate; - } gmm_u; -} WIRE_ATTR gmnal_msg_t; - -#define GMNAL_MSG_MAGIC LNET_PROTO_GM_MAGIC -#define GMNAL_MSG_VERSION 1 -#define GMNAL_MSG_IMMEDIATE 1 - -typedef struct netbuf { - __u64 nb_netaddr; /* network VM address */ - lnet_kiov_t nb_kiov[1]; /* the pages (at least 1) */ -} gmnal_netbuf_t; - -#define GMNAL_NETBUF_MSG(nb) ((gmnal_msg_t *)page_address((nb)->nb_kiov[0].kiov_page)) -#define GMNAL_NETBUF_LOCAL_NETADDR(nb) ((void *)((unsigned long)(nb)->nb_netaddr)) - -typedef struct gmnal_txbuf { - cfs_list_t txb_list; /* queue on gmni_idle_ltxbs */ - struct gmnal_txbuf *txb_next; /* stash on gmni_ltxs */ - gmnal_netbuf_t txb_buf; /* space */ -} gmnal_txbuf_t; - -typedef struct gmnal_tx { - cfs_list_t tx_list; /* queue */ - int tx_credit:1; /* consumed a credit? */ - int tx_large_iskiov:1; /* large is in kiovs? */ - struct gmnal_ni *tx_gmni; /* owning NI */ - lnet_nid_t tx_nid; /* destination NID */ - int tx_gmlid; /* destination GM local ID */ - lnet_msg_t *tx_lntmsg; /* lntmsg to finalize on completion */ - - gmnal_netbuf_t tx_buf; /* small tx buffer */ - gmnal_txbuf_t *tx_ltxb; /* large buffer (to free on completion) */ - int tx_msgnob; /* message size (so far) */ - - int tx_large_nob; /* # bytes large buffer payload */ - int tx_large_offset; /* offset within frags */ - int tx_large_niov; /* # VM frags */ - union { - struct iovec *iov; /* mapped frags */ - lnet_kiov_t *kiov; /* page frags */ - } tx_large_frags; - cfs_time_t tx_launchtime; /* when (in jiffies) the - * transmit was launched */ - struct gmnal_tx *tx_next; /* stash on gmni_txs */ -} gmnal_tx_t; - -typedef struct gmnal_rx { - cfs_list_t rx_list; /* enqueue on gmni_rxq for handling */ - int rx_islarge:1; /* large receive buffer? */ - unsigned int rx_recv_nob; /* bytes received */ - __u16 rx_recv_gmid; /* sender */ - __u8 rx_recv_port; /* sender's port */ - __u8 rx_recv_type; /* ?? */ - struct gmnal_rx *rx_next; /* stash on gmni_rxs */ - gmnal_netbuf_t rx_buf; /* the buffer */ -} gmnal_rx_t; - -typedef struct gmnal_ni { - lnet_ni_t *gmni_ni; /* generic NI */ - struct gm_port *gmni_port; /* GM port */ - cfs_spinlock_t gmni_gm_lock; /* serialise GM calls */ - int gmni_large_pages; /* # pages in a large message buffer */ - int gmni_large_msgsize;/* nob in large message buffers */ - int gmni_large_gmsize; /* large message GM bucket */ - int gmni_small_msgsize;/* nob in small message buffers */ - int gmni_small_gmsize; /* small message GM bucket */ - __u64 gmni_netaddr_base; /* base of mapped network VM */ - int gmni_netaddr_size; /* # bytes of mapped network VM */ - - gmnal_tx_t *gmni_txs; /* all txs */ - gmnal_rx_t *gmni_rxs; /* all rx descs */ - gmnal_txbuf_t *gmni_ltxbs; /* all large tx bufs */ - - cfs_atomic_t gmni_nthreads; /* total # threads */ - gm_alarm_t gmni_alarm; /* alarm to wake caretaker */ - int gmni_shutdown; /* tell all threads to exit */ - - cfs_list_t gmni_idle_txs; /* idle tx's */ - int gmni_tx_credits; /* # transmits still possible */ - cfs_list_t gmni_idle_ltxbs; /* idle large tx buffers */ - cfs_list_t gmni_buf_txq; /* tx's waiting for buffers */ - cfs_list_t gmni_cred_txq; /* tx's waiting for credits */ - cfs_spinlock_t gmni_tx_lock; /* serialise */ - - struct gm_hash *gmni_rx_hash; /* buffer->rx lookup */ - cfs_semaphore_t gmni_rx_mutex; /* serialise blocking on GM */ -} gmnal_ni_t; - -typedef struct { - int *gm_port; - int *gm_ntx; - int *gm_credits; - int *gm_peer_credits; - int *gm_nlarge_tx_bufs; - int *gm_nrx_small; - int *gm_nrx_large; - -#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM - cfs_sysctl_table_header_t *gm_sysctl; /* sysctl interface */ -#endif -} gmnal_tunables_t; - - -/* gmnal_api.c */ -int gmnal_init(void); -void gmnal_fini(void); -int gmnal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg); -int gmnal_startup(lnet_ni_t *ni); -void gmnal_shutdown(lnet_ni_t *ni); - -/* gmnal_cb.c */ -int gmnal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, - int delayed, unsigned int niov, - struct iovec *iov, lnet_kiov_t *kiov, - unsigned int offset, unsigned int mlen, unsigned int rlen); -int gmnal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg); - -/* gmnal_util.c */ -void gmnal_free_ltxbufs(gmnal_ni_t *gmni); -int gmnal_alloc_ltxbufs(gmnal_ni_t *gmni); -void gmnal_free_txs(gmnal_ni_t *gmni); -int gmnal_alloc_txs(gmnal_ni_t *gmni); -void gmnal_free_rxs(gmnal_ni_t *gmni); -int gmnal_alloc_rxs(gmnal_ni_t *gmni); -char *gmnal_gmstatus2str(gm_status_t status); -char *gmnal_rxevent2str(gm_recv_event_t *ev); -void gmnal_yield(int delay); - -/* gmnal_comm.c */ -void gmnal_post_rx(gmnal_ni_t *gmni, gmnal_rx_t *rx); -gmnal_tx_t *gmnal_get_tx(gmnal_ni_t *gmni); -void gmnal_tx_done(gmnal_tx_t *tx, int rc); -void gmnal_pack_msg(gmnal_ni_t *gmni, gmnal_msg_t *msg, - lnet_nid_t dstnid, int type); -void gmnal_stop_threads(gmnal_ni_t *gmni); -int gmnal_start_threads(gmnal_ni_t *gmni); -void gmnal_check_txqueues_locked (gmnal_ni_t *gmni); - -/* Module Parameters */ -extern gmnal_tunables_t gmnal_tunables; - -#endif /*__INCLUDE_GMNAL_H__*/ diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c deleted file mode 100644 index e8f55f1..0000000 --- a/lnet/klnds/gmlnd/gmlnd_api.c +++ /dev/null @@ -1,282 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -/* - * Implements the API NAL functions - */ - -#include "gmlnd.h" - -lnd_t the_gmlnd = -{ - .lnd_type = GMLND, - .lnd_startup = gmnal_startup, - .lnd_shutdown = gmnal_shutdown, - .lnd_ctl = gmnal_ctl, - .lnd_send = gmnal_send, - .lnd_recv = gmnal_recv, -}; - -gmnal_ni_t *the_gmni = NULL; - -int -gmnal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg) -{ - struct libcfs_ioctl_data *data = arg; - - switch (cmd) { - case IOC_LIBCFS_REGISTER_MYNID: - if (data->ioc_nid == ni->ni_nid) - return 0; - - LASSERT (LNET_NIDNET(data->ioc_nid) == LNET_NIDNET(ni->ni_nid)); - - CERROR("obsolete IOC_LIBCFS_REGISTER_MYNID for %s(%s)\n", - libcfs_nid2str(data->ioc_nid), - libcfs_nid2str(ni->ni_nid)); - return 0; - - default: - return (-EINVAL); - } -} - -int -gmnal_set_local_nid (gmnal_ni_t *gmni) -{ - lnet_ni_t *ni = gmni->gmni_ni; - __u32 local_gmid; - __u32 global_gmid; - gm_status_t gm_status; - - /* Called before anything initialised: no need to lock */ - gm_status = gm_get_node_id(gmni->gmni_port, &local_gmid); - if (gm_status != GM_SUCCESS) - return 0; - - CDEBUG(D_NET, "Local node id is [%u]\n", local_gmid); - - gm_status = gm_node_id_to_global_id(gmni->gmni_port, - local_gmid, - &global_gmid); - if (gm_status != GM_SUCCESS) - return 0; - - CDEBUG(D_NET, "Global node id is [%u]\n", global_gmid); - - ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), global_gmid); - return 1; -} - -void -gmnal_shutdown(lnet_ni_t *ni) -{ - gmnal_ni_t *gmni = ni->ni_data; - - CDEBUG(D_TRACE, "gmnal_api_shutdown: gmni [%p]\n", gmni); - - LASSERT (gmni == the_gmni); - - /* stop processing messages */ - gmnal_stop_threads(gmni); - - /* stop all network callbacks */ - gm_close(gmni->gmni_port); - gmni->gmni_port = NULL; - - gm_finalize(); - - gmnal_free_ltxbufs(gmni); - gmnal_free_txs(gmni); - gmnal_free_rxs(gmni); - - LIBCFS_FREE(gmni, sizeof(*gmni)); - - the_gmni = NULL; - PORTAL_MODULE_UNUSE; -} - -int -gmnal_startup(lnet_ni_t *ni) -{ - gmnal_ni_t *gmni = NULL; - gmnal_rx_t *rx = NULL; - gm_status_t gm_status; - int rc; - - LASSERT (ni->ni_lnd == &the_gmlnd); - - ni->ni_maxtxcredits = *gmnal_tunables.gm_credits; - ni->ni_peertxcredits = *gmnal_tunables.gm_peer_credits; - - if (the_gmni != NULL) { - CERROR("Only 1 instance supported\n"); - return -EINVAL; - } - - LIBCFS_ALLOC(gmni, sizeof(*gmni)); - if (gmni == NULL) { - CERROR("can't allocate gmni\n"); - return -ENOMEM; - } - - ni->ni_data = gmni; - - memset(gmni, 0, sizeof(*gmni)); - gmni->gmni_ni = ni; - cfs_spin_lock_init(&gmni->gmni_tx_lock); - cfs_spin_lock_init(&gmni->gmni_gm_lock); - CFS_INIT_LIST_HEAD(&gmni->gmni_idle_txs); - CFS_INIT_LIST_HEAD(&gmni->gmni_idle_ltxbs); - CFS_INIT_LIST_HEAD(&gmni->gmni_buf_txq); - CFS_INIT_LIST_HEAD(&gmni->gmni_cred_txq); - cfs_sema_init(&gmni->gmni_rx_mutex, 1); - PORTAL_MODULE_USE; - - /* - * initialise the interface, - */ - CDEBUG(D_NET, "Calling gm_init\n"); - if (gm_init() != GM_SUCCESS) { - CERROR("call to gm_init failed\n"); - goto failed_0; - } - - CDEBUG(D_NET, "Calling gm_open with port [%d], version [%d]\n", - *gmnal_tunables.gm_port, GM_API_VERSION); - - gm_status = gm_open(&gmni->gmni_port, 0, *gmnal_tunables.gm_port, - "gmnal", GM_API_VERSION); - - if (gm_status != GM_SUCCESS) { - CERROR("Can't open GM port %d: %d (%s)\n", - *gmnal_tunables.gm_port, gm_status, - gmnal_gmstatus2str(gm_status)); - goto failed_1; - } - - CDEBUG(D_NET,"gm_open succeeded port[%p]\n",gmni->gmni_port); - - if (!gmnal_set_local_nid(gmni)) - goto failed_2; - - CDEBUG(D_NET, "portals_nid is %s\n", libcfs_nid2str(ni->ni_nid)); - - gmni->gmni_large_msgsize = - offsetof(gmnal_msg_t, gmm_u.immediate.gmim_payload[LNET_MAX_PAYLOAD]); - gmni->gmni_large_gmsize = - gm_min_size_for_length(gmni->gmni_large_msgsize); - gmni->gmni_large_pages = - (gmni->gmni_large_msgsize + PAGE_SIZE - 1)/PAGE_SIZE; - - gmni->gmni_small_msgsize = MIN(GM_MTU, PAGE_SIZE); - gmni->gmni_small_gmsize = - gm_min_size_for_length(gmni->gmni_small_msgsize); - - gmni->gmni_netaddr_base = GMNAL_NETADDR_BASE; - gmni->gmni_netaddr_size = 0; - - CDEBUG(D_NET, "Msg size %08x/%08x [%d/%d]\n", - gmni->gmni_large_msgsize, gmni->gmni_small_msgsize, - gmni->gmni_large_gmsize, gmni->gmni_small_gmsize); - - if (gmnal_alloc_rxs(gmni) != 0) { - CERROR("Failed to allocate rx descriptors\n"); - goto failed_2; - } - - if (gmnal_alloc_txs(gmni) != 0) { - CERROR("Failed to allocate tx descriptors\n"); - goto failed_2; - } - - if (gmnal_alloc_ltxbufs(gmni) != 0) { - CERROR("Failed to allocate large tx buffers\n"); - goto failed_2; - } - - rc = gmnal_start_threads(gmni); - if (rc != 0) { - CERROR("Can't start threads: %d\n", rc); - goto failed_2; - } - - /* Start listening */ - for (rx = gmni->gmni_rxs; rx != NULL; rx = rx->rx_next) - gmnal_post_rx(gmni, rx); - - the_gmni = gmni; - - CDEBUG(D_NET, "gmnal_init finished\n"); - return 0; - - failed_2: - gm_close(gmni->gmni_port); - gmni->gmni_port = NULL; - - failed_1: - gm_finalize(); - - failed_0: - /* safe to free descriptors after network has been shut down */ - gmnal_free_ltxbufs(gmni); - gmnal_free_txs(gmni); - gmnal_free_rxs(gmni); - - LIBCFS_FREE(gmni, sizeof(*gmni)); - PORTAL_MODULE_UNUSE; - - return -EIO; -} - -/* - * Called when module loaded - */ -int gmnal_init(void) -{ - lnet_register_lnd(&the_gmlnd); - return 0; -} - -/* - * Called when module removed - */ -void gmnal_fini() -{ - lnet_unregister_lnd(&the_gmlnd); -} diff --git a/lnet/klnds/gmlnd/gmlnd_cb.c b/lnet/klnds/gmlnd/gmlnd_cb.c deleted file mode 100644 index 8addb11..0000000 --- a/lnet/klnds/gmlnd/gmlnd_cb.c +++ /dev/null @@ -1,177 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -/* - * This file implements the nal cb functions - */ - - -#include "gmlnd.h" - -int -gmnal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, - int delayed, unsigned int niov, - struct iovec *iov, lnet_kiov_t *kiov, - unsigned int offset, unsigned int mlen, unsigned int rlen) -{ - gmnal_ni_t *gmni = ni->ni_data; - gmnal_rx_t *rx = (gmnal_rx_t*)private; - gmnal_msg_t *msg = GMNAL_NETBUF_MSG(&rx->rx_buf); - int npages = rx->rx_islarge ? gmni->gmni_large_pages : 1; - int payload_offset = offsetof(gmnal_msg_t, - gmm_u.immediate.gmim_payload[0]); - int nob = payload_offset + mlen; - - LASSERT (msg->gmm_type == GMNAL_MSG_IMMEDIATE); - LASSERT (iov == NULL || kiov == NULL); - - if (rx->rx_recv_nob < nob) { - CERROR("Short message from nid %s: got %d, need %d\n", - libcfs_nid2str(msg->gmm_srcnid), rx->rx_recv_nob, nob); - gmnal_post_rx(gmni, rx); - return -EIO; - } - - if (kiov != NULL) - lnet_copy_kiov2kiov(niov, kiov, offset, - npages, rx->rx_buf.nb_kiov, payload_offset, - mlen); - else - lnet_copy_kiov2iov(niov, iov, offset, - npages, rx->rx_buf.nb_kiov, payload_offset, - mlen); - - lnet_finalize(ni, lntmsg, 0); - gmnal_post_rx(gmni, rx); - return 0; -} - -int -gmnal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg) -{ - lnet_hdr_t *hdr= &lntmsg->msg_hdr; - int type = lntmsg->msg_type; - lnet_process_id_t target = lntmsg->msg_target; - unsigned int niov = lntmsg->msg_niov; - struct iovec *iov = lntmsg->msg_iov; - lnet_kiov_t *kiov = lntmsg->msg_kiov; - unsigned int offset = lntmsg->msg_offset; - unsigned int len = lntmsg->msg_len; - gmnal_ni_t *gmni = ni->ni_data; - gm_status_t gmrc; - gmnal_tx_t *tx; - - LASSERT (iov == NULL || kiov == NULL); - - /* I may not block for a tx if I'm responding to an incoming message */ - tx = gmnal_get_tx(gmni); - if (tx == NULL) { - if (!gmni->gmni_shutdown) - CERROR ("Can't get tx for msg type %d for %s\n", - type, libcfs_nid2str(target.nid)); - return -EIO; - } - - tx->tx_nid = target.nid; - - gmrc = gm_global_id_to_node_id(gmni->gmni_port, LNET_NIDADDR(target.nid), - &tx->tx_gmlid); - if (gmrc != GM_SUCCESS) { - CERROR("Can't map Nid %s to a GM local ID: %d\n", - libcfs_nid2str(target.nid), gmrc); - /* NB tx_lntmsg not set => doesn't finalize */ - gmnal_tx_done(tx, -EIO); - return -EIO; - } - - gmnal_pack_msg(gmni, GMNAL_NETBUF_MSG(&tx->tx_buf), - target.nid, GMNAL_MSG_IMMEDIATE); - GMNAL_NETBUF_MSG(&tx->tx_buf)->gmm_u.immediate.gmim_hdr = *hdr; - tx->tx_msgnob = offsetof(gmnal_msg_t, gmm_u.immediate.gmim_payload[0]); - - if (the_lnet.ln_testprotocompat != 0) { - /* single-shot proto test */ - LNET_LOCK(); - if ((the_lnet.ln_testprotocompat & 1) != 0) { - GMNAL_NETBUF_MSG(&tx->tx_buf)->gmm_version++; - the_lnet.ln_testprotocompat &= ~1; - } - if ((the_lnet.ln_testprotocompat & 2) != 0) { - GMNAL_NETBUF_MSG(&tx->tx_buf)->gmm_magic = - LNET_PROTO_MAGIC; - the_lnet.ln_testprotocompat &= ~2; - } - LNET_UNLOCK(); - } - - if (tx->tx_msgnob + len <= gmni->gmni_small_msgsize) { - /* whole message fits in tx_buf */ - char *buffer = &(GMNAL_NETBUF_MSG(&tx->tx_buf)->gmm_u.immediate.gmim_payload[0]); - - if (iov != NULL) - lnet_copy_iov2flat(len, buffer, 0, - niov, iov, offset, len); - else - lnet_copy_kiov2flat(len, buffer, 0, - niov, kiov, offset, len); - - tx->tx_msgnob += len; - tx->tx_large_nob = 0; - } else { - /* stash payload pts to copy later */ - tx->tx_large_nob = len; - tx->tx_large_iskiov = (kiov != NULL); - tx->tx_large_niov = niov; - if (tx->tx_large_iskiov) - tx->tx_large_frags.kiov = kiov; - else - tx->tx_large_frags.iov = iov; - } - - LASSERT(tx->tx_lntmsg == NULL); - tx->tx_lntmsg = lntmsg; - - cfs_spin_lock(&gmni->gmni_tx_lock); - - cfs_list_add_tail(&tx->tx_list, &gmni->gmni_buf_txq); - gmnal_check_txqueues_locked(gmni); - - cfs_spin_unlock(&gmni->gmni_tx_lock); - - return 0; -} diff --git a/lnet/klnds/gmlnd/gmlnd_comm.c b/lnet/klnds/gmlnd/gmlnd_comm.c deleted file mode 100644 index bd74b98..0000000 --- a/lnet/klnds/gmlnd/gmlnd_comm.c +++ /dev/null @@ -1,580 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -/* - * This file contains all gmnal send and receive functions - */ - -#include "gmlnd.h" - -void -gmnal_notify_peer_down(gmnal_tx_t *tx) -{ - time_t then; - - then = cfs_time_current_sec() - - cfs_duration_sec(cfs_time_current() - - tx->tx_launchtime); - - lnet_notify(tx->tx_gmni->gmni_ni, tx->tx_nid, 0, then); -} - -void -gmnal_pack_msg(gmnal_ni_t *gmni, gmnal_msg_t *msg, - lnet_nid_t dstnid, int type) -{ - /* CAVEAT EMPTOR! this only sets the common message fields. */ - msg->gmm_magic = GMNAL_MSG_MAGIC; - msg->gmm_version = GMNAL_MSG_VERSION; - msg->gmm_type = type; - msg->gmm_srcnid = gmni->gmni_ni->ni_nid; - msg->gmm_dstnid = dstnid; -} - -int -gmnal_unpack_msg(gmnal_ni_t *gmni, gmnal_rx_t *rx) -{ - gmnal_msg_t *msg = GMNAL_NETBUF_MSG(&rx->rx_buf); - const int hdr_size = offsetof(gmnal_msg_t, gmm_u); - int buffnob = rx->rx_islarge ? gmni->gmni_large_msgsize : - gmni->gmni_small_msgsize; - int flip; - - /* rc = 0:SUCCESS -ve:failure +ve:version mismatch */ - - /* GM may not overflow our buffer */ - LASSERT (rx->rx_recv_nob <= buffnob); - - /* 6 bytes are enough to have received magic + version */ - if (rx->rx_recv_nob < 6) { - CERROR("Short message from gmid %u: %d\n", - rx->rx_recv_gmid, rx->rx_recv_nob); - return -EPROTO; - } - - if (msg->gmm_magic == GMNAL_MSG_MAGIC) { - flip = 0; - } else if (msg->gmm_magic == __swab32(GMNAL_MSG_MAGIC)) { - flip = 1; - } else if (msg->gmm_magic == LNET_PROTO_MAGIC || - msg->gmm_magic == __swab32(LNET_PROTO_MAGIC)) { - return EPROTO; - } else { - CERROR("Bad magic from gmid %u: %08x\n", - rx->rx_recv_gmid, msg->gmm_magic); - return -EPROTO; - } - - if (msg->gmm_version != - (flip ? __swab16(GMNAL_MSG_VERSION) : GMNAL_MSG_VERSION)) { - return EPROTO; - } - - if (rx->rx_recv_nob < hdr_size) { - CERROR("Short message from %u: %d\n", - rx->rx_recv_gmid, rx->rx_recv_nob); - return -EPROTO; - } - - if (flip) { - /* leave magic unflipped as a clue to peer endianness */ - __swab16s(&msg->gmm_version); - __swab16s(&msg->gmm_type); - __swab64s(&msg->gmm_srcnid); - __swab64s(&msg->gmm_dstnid); - } - - if (msg->gmm_srcnid == LNET_NID_ANY) { - CERROR("Bad src nid from %u: %s\n", - rx->rx_recv_gmid, libcfs_nid2str(msg->gmm_srcnid)); - return -EPROTO; - } - - if (gmni->gmni_ni->ni_nid != msg->gmm_dstnid) { - CERROR("Bad dst nid from %u: %s\n", - rx->rx_recv_gmid, libcfs_nid2str(msg->gmm_dstnid)); - return -EPROTO; - } - - switch (msg->gmm_type) { - default: - CERROR("Unknown message type from %u: %x\n", - rx->rx_recv_gmid, msg->gmm_type); - return -EPROTO; - - case GMNAL_MSG_IMMEDIATE: - if (rx->rx_recv_nob < - offsetof(gmnal_msg_t, gmm_u.immediate.gmim_payload[0])) { - CERROR("Short IMMEDIATE from %u: %d(%lu)\n", - rx->rx_recv_gmid, rx->rx_recv_nob, - (long)offsetof(gmnal_msg_t, - gmm_u.immediate.gmim_payload[0])); - return -EPROTO; - } - break; - } - return 0; -} - -gmnal_tx_t * -gmnal_get_tx(gmnal_ni_t *gmni) -{ - gmnal_tx_t *tx = NULL; - - cfs_spin_lock(&gmni->gmni_tx_lock); - - if (gmni->gmni_shutdown || - cfs_list_empty(&gmni->gmni_idle_txs)) { - cfs_spin_unlock(&gmni->gmni_tx_lock); - return NULL; - } - - tx = cfs_list_entry(gmni->gmni_idle_txs.next, gmnal_tx_t, tx_list); - cfs_list_del(&tx->tx_list); - - cfs_spin_unlock(&gmni->gmni_tx_lock); - - LASSERT (tx->tx_lntmsg == NULL); - LASSERT (tx->tx_ltxb == NULL); - LASSERT (!tx->tx_credit); - - return tx; -} - -void -gmnal_tx_done(gmnal_tx_t *tx, int rc) -{ - gmnal_ni_t *gmni = tx->tx_gmni; - int wake_sched = 0; - lnet_msg_t *lnetmsg = tx->tx_lntmsg; - - tx->tx_lntmsg = NULL; - - cfs_spin_lock(&gmni->gmni_tx_lock); - - if (tx->tx_ltxb != NULL) { - wake_sched = 1; - cfs_list_add_tail(&tx->tx_ltxb->txb_list, - &gmni->gmni_idle_ltxbs); - tx->tx_ltxb = NULL; - } - - if (tx->tx_credit) { - wake_sched = 1; - gmni->gmni_tx_credits++; - tx->tx_credit = 0; - } - - cfs_list_add_tail(&tx->tx_list, &gmni->gmni_idle_txs); - - if (wake_sched) - gmnal_check_txqueues_locked(gmni); - - cfs_spin_unlock(&gmni->gmni_tx_lock); - - /* Delay finalize until tx is free */ - if (lnetmsg != NULL) - lnet_finalize(gmni->gmni_ni, lnetmsg, rc); -} - -void -gmnal_drop_sends_callback(struct gm_port *gm_port, void *context, - gm_status_t status) -{ - gmnal_tx_t *tx = (gmnal_tx_t*)context; - - LASSERT(!cfs_in_interrupt()); - - CDEBUG(D_NET, "status for tx [%p] is [%d][%s], nid %s\n", - tx, status, gmnal_gmstatus2str(status), - libcfs_nid2str(tx->tx_nid)); - - gmnal_tx_done(tx, -EIO); -} - -void -gmnal_tx_callback(gm_port_t *gm_port, void *context, gm_status_t status) -{ - gmnal_tx_t *tx = (gmnal_tx_t*)context; - gmnal_ni_t *gmni = tx->tx_gmni; - - LASSERT(!cfs_in_interrupt()); - - switch(status) { - case GM_SUCCESS: - gmnal_tx_done(tx, 0); - return; - - case GM_SEND_DROPPED: - CDEBUG(D_NETERROR, "Dropped tx %p to %s\n", - tx, libcfs_nid2str(tx->tx_nid)); - /* Another tx failed and called gm_drop_sends() which made this - * one complete immediately */ - gmnal_tx_done(tx, -EIO); - return; - - default: - /* Some error; NB don't complete tx yet; we need its credit for - * gm_drop_sends() */ - CDEBUG(D_NETERROR, "tx %p error %d(%s), nid %s\n", - tx, status, gmnal_gmstatus2str(status), - libcfs_nid2str(tx->tx_nid)); - - gmnal_notify_peer_down(tx); - - cfs_spin_lock(&gmni->gmni_gm_lock); - gm_drop_sends(gmni->gmni_port, - tx->tx_ltxb != NULL ? - GMNAL_LARGE_PRIORITY : GMNAL_SMALL_PRIORITY, - tx->tx_gmlid, *gmnal_tunables.gm_port, - gmnal_drop_sends_callback, tx); - cfs_spin_unlock(&gmni->gmni_gm_lock); - return; - } - - /* not reached */ - LBUG(); -} - -void -gmnal_check_txqueues_locked (gmnal_ni_t *gmni) -{ - gmnal_tx_t *tx; - gmnal_txbuf_t *ltxb; - int gmsize; - int pri; - void *netaddr; - - tx = cfs_list_empty(&gmni->gmni_buf_txq) ? NULL : - cfs_list_entry(gmni->gmni_buf_txq.next, gmnal_tx_t, tx_list); - - if (tx != NULL && - (tx->tx_large_nob == 0 || - !cfs_list_empty(&gmni->gmni_idle_ltxbs))) { - - /* consume tx */ - cfs_list_del(&tx->tx_list); - - LASSERT (tx->tx_ltxb == NULL); - - if (tx->tx_large_nob != 0) { - ltxb = cfs_list_entry(gmni->gmni_idle_ltxbs.next, - gmnal_txbuf_t, txb_list); - - /* consume large buffer */ - cfs_list_del(<xb->txb_list); - - cfs_spin_unlock(&gmni->gmni_tx_lock); - - /* Unlocking here allows sends to get re-ordered, - * but we want to allow other CPUs to progress... */ - - tx->tx_ltxb = ltxb; - - /* marshall message in tx_ltxb... - * 1. Copy what was marshalled so far (in tx_buf) */ - memcpy(GMNAL_NETBUF_MSG(<xb->txb_buf), - GMNAL_NETBUF_MSG(&tx->tx_buf), tx->tx_msgnob); - - /* 2. Copy the payload */ - if (tx->tx_large_iskiov) - lnet_copy_kiov2kiov( - gmni->gmni_large_pages, - ltxb->txb_buf.nb_kiov, - tx->tx_msgnob, - tx->tx_large_niov, - tx->tx_large_frags.kiov, - tx->tx_large_offset, - tx->tx_large_nob); - else - lnet_copy_iov2kiov( - gmni->gmni_large_pages, - ltxb->txb_buf.nb_kiov, - tx->tx_msgnob, - tx->tx_large_niov, - tx->tx_large_frags.iov, - tx->tx_large_offset, - tx->tx_large_nob); - - tx->tx_msgnob += tx->tx_large_nob; - - cfs_spin_lock(&gmni->gmni_tx_lock); - } - - cfs_list_add_tail(&tx->tx_list, &gmni->gmni_cred_txq); - } - - if (!cfs_list_empty(&gmni->gmni_cred_txq) && - gmni->gmni_tx_credits != 0) { - - tx = cfs_list_entry(gmni->gmni_cred_txq.next, gmnal_tx_t, - tx_list); - - /* consume tx and 1 credit */ - cfs_list_del(&tx->tx_list); - gmni->gmni_tx_credits--; - - cfs_spin_unlock(&gmni->gmni_tx_lock); - - /* Unlocking here allows sends to get re-ordered, but we want - * to allow other CPUs to progress... */ - - LASSERT(!tx->tx_credit); - tx->tx_credit = 1; - - tx->tx_launchtime = cfs_time_current(); - - if (tx->tx_msgnob <= gmni->gmni_small_msgsize) { - LASSERT (tx->tx_ltxb == NULL); - netaddr = GMNAL_NETBUF_LOCAL_NETADDR(&tx->tx_buf); - gmsize = gmni->gmni_small_gmsize; - pri = GMNAL_SMALL_PRIORITY; - } else { - LASSERT (tx->tx_ltxb != NULL); - netaddr = GMNAL_NETBUF_LOCAL_NETADDR(&tx->tx_ltxb->txb_buf); - gmsize = gmni->gmni_large_gmsize; - pri = GMNAL_LARGE_PRIORITY; - } - - cfs_spin_lock(&gmni->gmni_gm_lock); - - gm_send_to_peer_with_callback(gmni->gmni_port, - netaddr, gmsize, - tx->tx_msgnob, - pri, - tx->tx_gmlid, - gmnal_tx_callback, - (void*)tx); - - cfs_spin_unlock(&gmni->gmni_gm_lock); - cfs_spin_lock(&gmni->gmni_tx_lock); - } -} - -void -gmnal_post_rx(gmnal_ni_t *gmni, gmnal_rx_t *rx) -{ - int gmsize = rx->rx_islarge ? gmni->gmni_large_gmsize : - gmni->gmni_small_gmsize; - int pri = rx->rx_islarge ? GMNAL_LARGE_PRIORITY : - GMNAL_SMALL_PRIORITY; - void *buffer = GMNAL_NETBUF_LOCAL_NETADDR(&rx->rx_buf); - - CDEBUG(D_NET, "posting rx %p buf %p\n", rx, buffer); - - cfs_spin_lock(&gmni->gmni_gm_lock); - gm_provide_receive_buffer_with_tag(gmni->gmni_port, - buffer, gmsize, pri, 0); - cfs_spin_unlock(&gmni->gmni_gm_lock); -} - -void -gmnal_version_reply (gmnal_ni_t *gmni, gmnal_rx_t *rx) -{ - /* Future protocol version compatibility support! - * The next gmlnd-specific protocol rev will first send a message to - * check version; I reply with a stub message containing my current - * magic+version... */ - gmnal_msg_t *msg; - gmnal_tx_t *tx = gmnal_get_tx(gmni); - - if (tx == NULL) { - CERROR("Can't allocate tx to send version info to %u\n", - rx->rx_recv_gmid); - return; - } - - LASSERT (tx->tx_lntmsg == NULL); /* no finalize */ - - tx->tx_nid = LNET_NID_ANY; - tx->tx_gmlid = rx->rx_recv_gmid; - - msg = GMNAL_NETBUF_MSG(&tx->tx_buf); - msg->gmm_magic = GMNAL_MSG_MAGIC; - msg->gmm_version = GMNAL_MSG_VERSION; - - /* just send magic + version */ - tx->tx_msgnob = offsetof(gmnal_msg_t, gmm_type); - tx->tx_large_nob = 0; - - cfs_spin_lock(&gmni->gmni_tx_lock); - - cfs_list_add_tail(&tx->tx_list, &gmni->gmni_buf_txq); - gmnal_check_txqueues_locked(gmni); - - cfs_spin_unlock(&gmni->gmni_tx_lock); -} - -int -gmnal_rx_thread(void *arg) -{ - gmnal_ni_t *gmni = arg; - gm_recv_event_t *rxevent = NULL; - gm_recv_t *recv = NULL; - gmnal_rx_t *rx; - int rc; - - cfs_daemonize("gmnal_rxd"); - - while (!gmni->gmni_shutdown) { - rc = down_interruptible(&gmni->gmni_rx_mutex); - LASSERT (rc == 0 || rc == -EINTR); - if (rc != 0) - continue; - - cfs_spin_lock(&gmni->gmni_gm_lock); - rxevent = gm_blocking_receive_no_spin(gmni->gmni_port); - cfs_spin_unlock(&gmni->gmni_gm_lock); - - switch (GM_RECV_EVENT_TYPE(rxevent)) { - default: - gm_unknown(gmni->gmni_port, rxevent); - cfs_up(&gmni->gmni_rx_mutex); - continue; - - case GM_FAST_RECV_EVENT: - case GM_FAST_PEER_RECV_EVENT: - case GM_PEER_RECV_EVENT: - case GM_FAST_HIGH_RECV_EVENT: - case GM_FAST_HIGH_PEER_RECV_EVENT: - case GM_HIGH_PEER_RECV_EVENT: - case GM_RECV_EVENT: - case GM_HIGH_RECV_EVENT: - break; - } - - recv = &rxevent->recv; - rx = gm_hash_find(gmni->gmni_rx_hash, - gm_ntohp(recv->buffer)); - LASSERT (rx != NULL); - - rx->rx_recv_nob = gm_ntoh_u32(recv->length); - rx->rx_recv_gmid = gm_ntoh_u16(recv->sender_node_id); - rx->rx_recv_port = gm_ntoh_u8(recv->sender_port_id); - rx->rx_recv_type = gm_ntoh_u8(recv->type); - - switch (GM_RECV_EVENT_TYPE(rxevent)) { - case GM_FAST_RECV_EVENT: - case GM_FAST_PEER_RECV_EVENT: - case GM_FAST_HIGH_RECV_EVENT: - case GM_FAST_HIGH_PEER_RECV_EVENT: - LASSERT (rx->rx_recv_nob <= PAGE_SIZE); - - memcpy(GMNAL_NETBUF_MSG(&rx->rx_buf), - gm_ntohp(recv->message), rx->rx_recv_nob); - break; - } - - cfs_up(&gmni->gmni_rx_mutex); - - CDEBUG (D_NET, "rx %p: buf %p(%p) nob %d\n", rx, - GMNAL_NETBUF_LOCAL_NETADDR(&rx->rx_buf), - gm_ntohp(recv->buffer), rx->rx_recv_nob); - - /* We're connectionless: simply drop packets with - * errors */ - rc = gmnal_unpack_msg(gmni, rx); - - if (rc == 0) { - gmnal_msg_t *msg = GMNAL_NETBUF_MSG(&rx->rx_buf); - - LASSERT (msg->gmm_type == GMNAL_MSG_IMMEDIATE); - rc = lnet_parse(gmni->gmni_ni, - &msg->gmm_u.immediate.gmim_hdr, - msg->gmm_srcnid, rx, 0); - } else if (rc > 0) { - gmnal_version_reply(gmni, rx); - rc = -EPROTO; /* repost rx */ - } - - if (rc < 0) /* parse failure */ - gmnal_post_rx(gmni, rx); - } - - CDEBUG(D_NET, "exiting\n"); - cfs_atomic_dec(&gmni->gmni_nthreads); - return 0; -} - -void -gmnal_stop_threads(gmnal_ni_t *gmni) -{ - int count = 2; - - gmni->gmni_shutdown = 1; - cfs_mb(); - - /* wake rxthread owning gmni_rx_mutex with an alarm. */ - cfs_spin_lock(&gmni->gmni_gm_lock); - gm_set_alarm(gmni->gmni_port, &gmni->gmni_alarm, 0, NULL, NULL); - cfs_spin_unlock(&gmni->gmni_gm_lock); - - while (cfs_atomic_read(&gmni->gmni_nthreads) != 0) { - count++; - if ((count & (count - 1)) == 0) - CWARN("Waiting for %d threads to stop\n", - cfs_atomic_read(&gmni->gmni_nthreads)); - gmnal_yield(1); - } -} - -int -gmnal_start_threads(gmnal_ni_t *gmni) -{ - int i; - int pid; - - LASSERT (!gmni->gmni_shutdown); - LASSERT (cfs_atomic_read(&gmni->gmni_nthreads) == 0); - - gm_initialize_alarm(&gmni->gmni_alarm); - - for (i = 0; i < cfs_num_online_cpus(); i++) { - - pid = cfs_kernel_thread(gmnal_rx_thread, (void*)gmni, 0); - if (pid < 0) { - CERROR("rx thread failed to start: %d\n", pid); - gmnal_stop_threads(gmni); - return pid; - } - - cfs_atomic_inc(&gmni->gmni_nthreads); - } - - return 0; -} diff --git a/lnet/klnds/gmlnd/gmlnd_module.c b/lnet/klnds/gmlnd/gmlnd_module.c deleted file mode 100644 index 1dad4a0..0000000 --- a/lnet/klnds/gmlnd/gmlnd_module.c +++ /dev/null @@ -1,221 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -#include "gmlnd.h" - - -static int port = 4; -CFS_MODULE_PARM(port, "i", int, 0444, - "GM port to use for communications"); - -static int ntx = 256; -CFS_MODULE_PARM(ntx, "i", int, 0444, - "# tx descriptors"); - -static int credits = 128; -CFS_MODULE_PARM(credits, "i", int, 0444, - "# concurrent sends"); - -static int peer_credits = 8; -CFS_MODULE_PARM(peer_credits, "i", int, 0444, - "# concurrent sends per peer"); - -static int nlarge_tx_bufs = 32; -CFS_MODULE_PARM(nlarge_tx_bufs, "i", int, 0444, - "# large tx message buffers"); - -static int nrx_small = 128; -CFS_MODULE_PARM(nrx_small, "i", int, 0444, - "# small rx message buffers"); - -static int nrx_large = 64; -CFS_MODULE_PARM(nrx_large, "i", int, 0444, - "# large rx message buffers"); - -gmnal_tunables_t gmnal_tunables = { - .gm_port = &port, - .gm_ntx = &ntx, - .gm_credits = &credits, - .gm_peer_credits = &peer_credits, - .gm_nlarge_tx_bufs = &nlarge_tx_bufs, - .gm_nrx_small = &nrx_small, - .gm_nrx_large = &nrx_large, -}; - -#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM - -#ifndef HAVE_SYSCTL_UNNUMBERED - -enum { - GMLND_PORT = 1, - GMLND_NTX, - GMLND_CREDITS, - GMLND_PEERCREDITS, - GMLND_NLARGE_TX_BUFS, - GMLND_NRX_SMALL, - GMLND_NRX_LARGE -}; - -#else - -#define GMLND_PORT CTL_UNNUMBERED -#define GMLND_NTX CTL_UNNUMBERED -#define GMLND_CREDITS CTL_UNNUMBERED -#define GMLND_PEERCREDITS CTL_UNNUMBERED -#define GMLND_NLARGE_TX_BUFS CTL_UNNUMBERED -#define GMLND_NRX_SMALL CTL_UNNUMBERED -#define GMLND_NRX_LARGE CTL_UNNUMBERED - -#endif - -static cfs_sysctl_table_t gmnal_ctl_table[] = { - { - .ctl_name = GMLND_PORT, - .procname = "port", - .data = &port, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_NTX, - .procname = "ntx", - .data = &ntx, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_CREDITS, - .procname = "credits", - .data = &credits, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_PEERCREDITS, - .procname = "peer_credits", - .data = &peer_credits, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_NLARGE_TX_BUFS, - .procname = "nlarge_tx_bufs", - .data = &nlarge_tx_bufs, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_NRX_SMALL, - .procname = "nrx_small", - .data = &nrx_small, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = GMLND_NRX_LARGE, - .procname = "nrx_large", - .data = &nrx_large, - .maxlen = sizeof (int), - .mode = 0444, - .proc_handler = &proc_dointvec - }, - {0} -}; - -static cfs_sysctl_table_t gmnal_top_ctl_table[] = { - { - .ctl_name = CTL_GMLND, - .procname = "gmnal", - .data = NULL, - .maxlen = 0, - .mode = 0555, - .child = gmnal_ctl_table - }, - {0} -}; -#endif - -static int __init -gmnal_load(void) -{ - int status; - CDEBUG(D_TRACE, "This is the gmnal module initialisation routine\n"); - -#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM - gmnal_tunables.gm_sysctl = - cfs_register_sysctl_table(gmnal_top_ctl_table, 0); - - if (gmnal_tunables.gm_sysctl == NULL) - CWARN("Can't setup /proc tunables\n"); -#endif - CDEBUG(D_NET, "Calling gmnal_init\n"); - status = gmnal_init(); - if (status == 0) { - CDEBUG(D_NET, "Portals GMNAL initialised ok\n"); - } else { - CDEBUG(D_NET, "Portals GMNAL Failed to initialise\n"); - return(-ENODEV); - } - - CDEBUG(D_NET, "This is the end of the gmnal init routine"); - - return(0); -} - -static void __exit -gmnal_unload(void) -{ - gmnal_fini(); -#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM - if (gmnal_tunables.gm_sysctl != NULL) - cfs_unregister_sysctl_table(gmnal_tunables.gm_sysctl); -#endif -} - -module_init(gmnal_load); -module_exit(gmnal_unload); - -MODULE_AUTHOR("Sun Microsystems, Inc. "); -MODULE_DESCRIPTION("Kernel GM LND v1.01"); -MODULE_LICENSE("GPL"); diff --git a/lnet/klnds/gmlnd/gmlnd_utils.c b/lnet/klnds/gmlnd/gmlnd_utils.c deleted file mode 100644 index 3f9fdec..0000000 --- a/lnet/klnds/gmlnd/gmlnd_utils.c +++ /dev/null @@ -1,595 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -#include "gmlnd.h" - -void -gmnal_free_netbuf_pages (gmnal_netbuf_t *nb, int npages) -{ - int i; - - for (i = 0; i < npages; i++) - __free_page(nb->nb_kiov[i].kiov_page); -} - -int -gmnal_alloc_netbuf_pages (gmnal_ni_t *gmni, gmnal_netbuf_t *nb, int npages) -{ - int i; - gm_status_t gmrc; - - LASSERT (npages > 0); - - for (i = 0; i < npages; i++) { - nb->nb_kiov[i].kiov_page = alloc_page(GFP_KERNEL); - nb->nb_kiov[i].kiov_offset = 0; - nb->nb_kiov[i].kiov_len = PAGE_SIZE; - - if (nb->nb_kiov[i].kiov_page == NULL) { - CERROR("Can't allocate page\n"); - gmnal_free_netbuf_pages(nb, i); - return -ENOMEM; - } - - CDEBUG(D_NET,"[%3d] page %p, phys "LPX64", @ "LPX64"\n", - i, nb->nb_kiov[i].kiov_page, - lnet_page2phys(nb->nb_kiov[i].kiov_page), - gmni->gmni_netaddr_base); - - gmrc = gm_register_memory_ex_phys( - gmni->gmni_port, - lnet_page2phys(nb->nb_kiov[i].kiov_page), - PAGE_SIZE, - gmni->gmni_netaddr_base); - CDEBUG(D_NET,"[%3d] page %p: %d\n", - i, nb->nb_kiov[i].kiov_page, gmrc); - - if (gmrc != GM_SUCCESS) { - CERROR("Can't map page: %d(%s)\n", gmrc, - gmnal_gmstatus2str(gmrc)); - gmnal_free_netbuf_pages(nb, i+1); - return -ENOMEM; - } - - if (i == 0) - nb->nb_netaddr = gmni->gmni_netaddr_base; - - gmni->gmni_netaddr_base += PAGE_SIZE; - } - - return 0; -} - -void -gmnal_free_ltxbuf (gmnal_ni_t *gmni, gmnal_txbuf_t *txb) -{ - int npages = gmni->gmni_large_pages; - - LASSERT (gmni->gmni_port == NULL); - /* No unmapping; the port has been closed */ - - gmnal_free_netbuf_pages(&txb->txb_buf, gmni->gmni_large_pages); - LIBCFS_FREE(txb, offsetof(gmnal_txbuf_t, txb_buf.nb_kiov[npages])); -} - -int -gmnal_alloc_ltxbuf (gmnal_ni_t *gmni) -{ - int npages = gmni->gmni_large_pages; - int sz = offsetof(gmnal_txbuf_t, txb_buf.nb_kiov[npages]); - gmnal_txbuf_t *txb; - int rc; - - LIBCFS_ALLOC(txb, sz); - if (txb == NULL) { - CERROR("Can't allocate large txbuffer\n"); - return -ENOMEM; - } - - rc = gmnal_alloc_netbuf_pages(gmni, &txb->txb_buf, npages); - if (rc != 0) { - LIBCFS_FREE(txb, sz); - return rc; - } - - cfs_list_add_tail(&txb->txb_list, &gmni->gmni_idle_ltxbs); - - txb->txb_next = gmni->gmni_ltxbs; - gmni->gmni_ltxbs = txb; - - return 0; -} - -void -gmnal_free_tx (gmnal_tx_t *tx) -{ - LASSERT (tx->tx_gmni->gmni_port == NULL); - - gmnal_free_netbuf_pages(&tx->tx_buf, 1); - LIBCFS_FREE(tx, sizeof(*tx)); -} - -int -gmnal_alloc_tx (gmnal_ni_t *gmni) -{ - gmnal_tx_t *tx; - int rc; - - LIBCFS_ALLOC(tx, sizeof(*tx)); - if (tx == NULL) { - CERROR("Failed to allocate tx\n"); - return -ENOMEM; - } - - memset(tx, 0, sizeof(*tx)); - - rc = gmnal_alloc_netbuf_pages(gmni, &tx->tx_buf, 1); - if (rc != 0) { - LIBCFS_FREE(tx, sizeof(*tx)); - return -ENOMEM; - } - - tx->tx_gmni = gmni; - - cfs_list_add_tail(&tx->tx_list, &gmni->gmni_idle_txs); - - tx->tx_next = gmni->gmni_txs; - gmni->gmni_txs = tx; - - return 0; -} - -void -gmnal_free_rx(gmnal_ni_t *gmni, gmnal_rx_t *rx) -{ - int npages = rx->rx_islarge ? gmni->gmni_large_pages : 1; - - LASSERT (gmni->gmni_port == NULL); - - gmnal_free_netbuf_pages(&rx->rx_buf, npages); - LIBCFS_FREE(rx, offsetof(gmnal_rx_t, rx_buf.nb_kiov[npages])); -} - -int -gmnal_alloc_rx (gmnal_ni_t *gmni, int islarge) -{ - int npages = islarge ? gmni->gmni_large_pages : 1; - int sz = offsetof(gmnal_rx_t, rx_buf.nb_kiov[npages]); - int rc; - gmnal_rx_t *rx; - gm_status_t gmrc; - - LIBCFS_ALLOC(rx, sz); - if (rx == NULL) { - CERROR("Failed to allocate rx\n"); - return -ENOMEM; - } - - memset(rx, 0, sizeof(*rx)); - - rc = gmnal_alloc_netbuf_pages(gmni, &rx->rx_buf, npages); - if (rc != 0) { - LIBCFS_FREE(rx, sz); - return rc; - } - - rx->rx_islarge = islarge; - rx->rx_next = gmni->gmni_rxs; - gmni->gmni_rxs = rx; - - gmrc = gm_hash_insert(gmni->gmni_rx_hash, - GMNAL_NETBUF_LOCAL_NETADDR(&rx->rx_buf), rx); - if (gmrc != GM_SUCCESS) { - CERROR("Couldn't add rx to hash table: %d\n", gmrc); - return -ENOMEM; - } - - return 0; -} - -void -gmnal_free_ltxbufs (gmnal_ni_t *gmni) -{ - gmnal_txbuf_t *txb; - - while ((txb = gmni->gmni_ltxbs) != NULL) { - gmni->gmni_ltxbs = txb->txb_next; - gmnal_free_ltxbuf(gmni, txb); - } -} - -int -gmnal_alloc_ltxbufs (gmnal_ni_t *gmni) -{ - int nlarge_tx_bufs = *gmnal_tunables.gm_nlarge_tx_bufs; - int i; - int rc; - - for (i = 0; i < nlarge_tx_bufs; i++) { - rc = gmnal_alloc_ltxbuf(gmni); - - if (rc != 0) - return rc; - } - - return 0; -} - -void -gmnal_free_txs(gmnal_ni_t *gmni) -{ - gmnal_tx_t *tx; - - while ((tx = gmni->gmni_txs) != NULL) { - gmni->gmni_txs = tx->tx_next; - gmnal_free_tx (tx); - } -} - -int -gmnal_alloc_txs(gmnal_ni_t *gmni) -{ - int ntxcred = gm_num_send_tokens(gmni->gmni_port); - int ntx = *gmnal_tunables.gm_ntx; - int i; - int rc; - - CDEBUG(D_NET, "ntxcred: %d\n", ntxcred); - gmni->gmni_tx_credits = ntxcred; - - for (i = 0; i < ntx; i++) { - rc = gmnal_alloc_tx(gmni); - if (rc != 0) - return rc; - } - - return 0; -} - -void -gmnal_free_rxs(gmnal_ni_t *gmni) -{ - gmnal_rx_t *rx; - - while ((rx = gmni->gmni_rxs) != NULL) { - gmni->gmni_rxs = rx->rx_next; - - gmnal_free_rx(gmni, rx); - } - - LASSERT (gmni->gmni_port == NULL); -#if 0 - /* GM releases all resources allocated to a port when it closes */ - if (gmni->gmni_rx_hash != NULL) - gm_destroy_hash(gmni->gmni_rx_hash); -#endif -} - -int -gmnal_alloc_rxs (gmnal_ni_t *gmni) -{ - int nrxcred = gm_num_receive_tokens(gmni->gmni_port); - int nrx_small = *gmnal_tunables.gm_nrx_small; - int nrx_large = *gmnal_tunables.gm_nrx_large; - int nrx = nrx_large + nrx_small; - int rc; - int i; - - CDEBUG(D_NET, "nrxcred: %d(%dL+%dS)\n", nrxcred, nrx_large, nrx_small); - - if (nrx > nrxcred) { - int nlarge = (nrx_large * nrxcred)/nrx; - int nsmall = nrxcred - nlarge; - - CWARN("Only %d rx credits: " - "reducing large %d->%d, small %d->%d\n", nrxcred, - nrx_large, nlarge, nrx_small, nsmall); - - *gmnal_tunables.gm_nrx_large = nrx_large = nlarge; - *gmnal_tunables.gm_nrx_small = nrx_small = nsmall; - nrx = nlarge + nsmall; - } - - gmni->gmni_rx_hash = gm_create_hash(gm_hash_compare_ptrs, - gm_hash_hash_ptr, 0, 0, nrx, 0); - if (gmni->gmni_rx_hash == NULL) { - CERROR("Failed to create hash table\n"); - return -ENOMEM; - } - - for (i = 0; i < nrx; i++ ) { - rc = gmnal_alloc_rx(gmni, i < nrx_large); - if (rc != 0) - return rc; - } - - return 0; -} - -char * -gmnal_gmstatus2str(gm_status_t status) -{ - return(gm_strerror(status)); - - switch(status) { - case(GM_SUCCESS): - return("SUCCESS"); - case(GM_FAILURE): - return("FAILURE"); - case(GM_INPUT_BUFFER_TOO_SMALL): - return("INPUT_BUFFER_TOO_SMALL"); - case(GM_OUTPUT_BUFFER_TOO_SMALL): - return("OUTPUT_BUFFER_TOO_SMALL"); - case(GM_TRY_AGAIN ): - return("TRY_AGAIN"); - case(GM_BUSY): - return("BUSY"); - case(GM_MEMORY_FAULT): - return("MEMORY_FAULT"); - case(GM_INTERRUPTED): - return("INTERRUPTED"); - case(GM_INVALID_PARAMETER): - return("INVALID_PARAMETER"); - case(GM_OUT_OF_MEMORY): - return("OUT_OF_MEMORY"); - case(GM_INVALID_COMMAND): - return("INVALID_COMMAND"); - case(GM_PERMISSION_DENIED): - return("PERMISSION_DENIED"); - case(GM_INTERNAL_ERROR): - return("INTERNAL_ERROR"); - case(GM_UNATTACHED): - return("UNATTACHED"); - case(GM_UNSUPPORTED_DEVICE): - return("UNSUPPORTED_DEVICE"); - case(GM_SEND_TIMED_OUT): - return("GM_SEND_TIMEDOUT"); - case(GM_SEND_REJECTED): - return("GM_SEND_REJECTED"); - case(GM_SEND_TARGET_PORT_CLOSED): - return("GM_SEND_TARGET_PORT_CLOSED"); - case(GM_SEND_TARGET_NODE_UNREACHABLE): - return("GM_SEND_TARGET_NODE_UNREACHABLE"); - case(GM_SEND_DROPPED): - return("GM_SEND_DROPPED"); - case(GM_SEND_PORT_CLOSED): - return("GM_SEND_PORT_CLOSED"); - case(GM_NODE_ID_NOT_YET_SET): - return("GM_NODE_ID_NOT_YET_SET"); - case(GM_STILL_SHUTTING_DOWN): - return("GM_STILL_SHUTTING_DOWN"); - case(GM_CLONE_BUSY): - return("GM_CLONE_BUSY"); - case(GM_NO_SUCH_DEVICE): - return("GM_NO_SUCH_DEVICE"); - case(GM_ABORTED): - return("GM_ABORTED"); - case(GM_INCOMPATIBLE_LIB_AND_DRIVER): - return("GM_INCOMPATIBLE_LIB_AND_DRIVER"); - case(GM_UNTRANSLATED_SYSTEM_ERROR): - return("GM_UNTRANSLATED_SYSTEM_ERROR"); - case(GM_ACCESS_DENIED): - return("GM_ACCESS_DENIED"); - - - /* - * These ones are in the docs but aren't in the header file - case(GM_DEV_NOT_FOUND): - return("GM_DEV_NOT_FOUND"); - case(GM_INVALID_PORT_NUMBER): - return("GM_INVALID_PORT_NUMBER"); - case(GM_UC_ERROR): - return("GM_US_ERROR"); - case(GM_PAGE_TABLE_FULL): - return("GM_PAGE_TABLE_FULL"); - case(GM_MINOR_OVERFLOW): - return("GM_MINOR_OVERFLOW"); - case(GM_SEND_ORPHANED): - return("GM_SEND_ORPHANED"); - case(GM_HARDWARE_FAULT): - return("GM_HARDWARE_FAULT"); - case(GM_DATA_CORRUPTED): - return("GM_DATA_CORRUPTED"); - case(GM_TIMED_OUT): - return("GM_TIMED_OUT"); - case(GM_USER_ERROR): - return("GM_USER_ERROR"); - case(GM_NO_MATCH): - return("GM_NOMATCH"); - case(GM_NOT_SUPPORTED_IN_KERNEL): - return("GM_NOT_SUPPORTED_IN_KERNEL"); - case(GM_NOT_SUPPORTED_ON_ARCH): - return("GM_NOT_SUPPORTED_ON_ARCH"); - case(GM_PTE_REF_CNT_OVERFLOW): - return("GM_PTR_REF_CNT_OVERFLOW"); - case(GM_NO_DRIVER_SUPPORT): - return("GM_NO_DRIVER_SUPPORT"); - case(GM_FIRMWARE_NOT_RUNNING): - return("GM_FIRMWARE_NOT_RUNNING"); - * These ones are in the docs but aren't in the header file - */ - - default: - return("UNKNOWN GM ERROR CODE"); - } -} - - -char * -gmnal_rxevent2str(gm_recv_event_t *ev) -{ - short event; - event = GM_RECV_EVENT_TYPE(ev); - switch(event) { - case(GM_NO_RECV_EVENT): - return("GM_NO_RECV_EVENT"); - case(GM_SENDS_FAILED_EVENT): - return("GM_SEND_FAILED_EVENT"); - case(GM_ALARM_EVENT): - return("GM_ALARM_EVENT"); - case(GM_SENT_EVENT): - return("GM_SENT_EVENT"); - case(_GM_SLEEP_EVENT): - return("_GM_SLEEP_EVENT"); - case(GM_RAW_RECV_EVENT): - return("GM_RAW_RECV_EVENT"); - case(GM_BAD_SEND_DETECTED_EVENT): - return("GM_BAD_SEND_DETECTED_EVENT"); - case(GM_SEND_TOKEN_VIOLATION_EVENT): - return("GM_SEND_TOKEN_VIOLATION_EVENT"); - case(GM_RECV_TOKEN_VIOLATION_EVENT): - return("GM_RECV_TOKEN_VIOLATION_EVENT"); - case(GM_BAD_RECV_TOKEN_EVENT): - return("GM_BAD_RECV_TOKEN_EVENT"); - case(GM_ALARM_VIOLATION_EVENT): - return("GM_ALARM_VIOLATION_EVENT"); - case(GM_RECV_EVENT): - return("GM_RECV_EVENT"); - case(GM_HIGH_RECV_EVENT): - return("GM_HIGH_RECV_EVENT"); - case(GM_PEER_RECV_EVENT): - return("GM_PEER_RECV_EVENT"); - case(GM_HIGH_PEER_RECV_EVENT): - return("GM_HIGH_PEER_RECV_EVENT"); - case(GM_FAST_RECV_EVENT): - return("GM_FAST_RECV_EVENT"); - case(GM_FAST_HIGH_RECV_EVENT): - return("GM_FAST_HIGH_RECV_EVENT"); - case(GM_FAST_PEER_RECV_EVENT): - return("GM_FAST_PEER_RECV_EVENT"); - case(GM_FAST_HIGH_PEER_RECV_EVENT): - return("GM_FAST_HIGH_PEER_RECV_EVENT"); - case(GM_REJECTED_SEND_EVENT): - return("GM_REJECTED_SEND_EVENT"); - case(GM_ORPHANED_SEND_EVENT): - return("GM_ORPHANED_SEND_EVENT"); - case(GM_BAD_RESEND_DETECTED_EVENT): - return("GM_BAD_RESEND_DETETED_EVENT"); - case(GM_DROPPED_SEND_EVENT): - return("GM_DROPPED_SEND_EVENT"); - case(GM_BAD_SEND_VMA_EVENT): - return("GM_BAD_SEND_VMA_EVENT"); - case(GM_BAD_RECV_VMA_EVENT): - return("GM_BAD_RECV_VMA_EVENT"); - case(_GM_FLUSHED_ALARM_EVENT): - return("GM_FLUSHED_ALARM_EVENT"); - case(GM_SENT_TOKENS_EVENT): - return("GM_SENT_TOKENS_EVENTS"); - case(GM_IGNORE_RECV_EVENT): - return("GM_IGNORE_RECV_EVENT"); - case(GM_ETHERNET_RECV_EVENT): - return("GM_ETHERNET_RECV_EVENT"); - case(GM_NEW_NO_RECV_EVENT): - return("GM_NEW_NO_RECV_EVENT"); - case(GM_NEW_SENDS_FAILED_EVENT): - return("GM_NEW_SENDS_FAILED_EVENT"); - case(GM_NEW_ALARM_EVENT): - return("GM_NEW_ALARM_EVENT"); - case(GM_NEW_SENT_EVENT): - return("GM_NEW_SENT_EVENT"); - case(_GM_NEW_SLEEP_EVENT): - return("GM_NEW_SLEEP_EVENT"); - case(GM_NEW_RAW_RECV_EVENT): - return("GM_NEW_RAW_RECV_EVENT"); - case(GM_NEW_BAD_SEND_DETECTED_EVENT): - return("GM_NEW_BAD_SEND_DETECTED_EVENT"); - case(GM_NEW_SEND_TOKEN_VIOLATION_EVENT): - return("GM_NEW_SEND_TOKEN_VIOLATION_EVENT"); - case(GM_NEW_RECV_TOKEN_VIOLATION_EVENT): - return("GM_NEW_RECV_TOKEN_VIOLATION_EVENT"); - case(GM_NEW_BAD_RECV_TOKEN_EVENT): - return("GM_NEW_BAD_RECV_TOKEN_EVENT"); - case(GM_NEW_ALARM_VIOLATION_EVENT): - return("GM_NEW_ALARM_VIOLATION_EVENT"); - case(GM_NEW_RECV_EVENT): - return("GM_NEW_RECV_EVENT"); - case(GM_NEW_HIGH_RECV_EVENT): - return("GM_NEW_HIGH_RECV_EVENT"); - case(GM_NEW_PEER_RECV_EVENT): - return("GM_NEW_PEER_RECV_EVENT"); - case(GM_NEW_HIGH_PEER_RECV_EVENT): - return("GM_NEW_HIGH_PEER_RECV_EVENT"); - case(GM_NEW_FAST_RECV_EVENT): - return("GM_NEW_FAST_RECV_EVENT"); - case(GM_NEW_FAST_HIGH_RECV_EVENT): - return("GM_NEW_FAST_HIGH_RECV_EVENT"); - case(GM_NEW_FAST_PEER_RECV_EVENT): - return("GM_NEW_FAST_PEER_RECV_EVENT"); - case(GM_NEW_FAST_HIGH_PEER_RECV_EVENT): - return("GM_NEW_FAST_HIGH_PEER_RECV_EVENT"); - case(GM_NEW_REJECTED_SEND_EVENT): - return("GM_NEW_REJECTED_SEND_EVENT"); - case(GM_NEW_ORPHANED_SEND_EVENT): - return("GM_NEW_ORPHANED_SEND_EVENT"); - case(_GM_NEW_PUT_NOTIFICATION_EVENT): - return("_GM_NEW_PUT_NOTIFICATION_EVENT"); - case(GM_NEW_FREE_SEND_TOKEN_EVENT): - return("GM_NEW_FREE_SEND_TOKEN_EVENT"); - case(GM_NEW_FREE_HIGH_SEND_TOKEN_EVENT): - return("GM_NEW_FREE_HIGH_SEND_TOKEN_EVENT"); - case(GM_NEW_BAD_RESEND_DETECTED_EVENT): - return("GM_NEW_BAD_RESEND_DETECTED_EVENT"); - case(GM_NEW_DROPPED_SEND_EVENT): - return("GM_NEW_DROPPED_SEND_EVENT"); - case(GM_NEW_BAD_SEND_VMA_EVENT): - return("GM_NEW_BAD_SEND_VMA_EVENT"); - case(GM_NEW_BAD_RECV_VMA_EVENT): - return("GM_NEW_BAD_RECV_VMA_EVENT"); - case(_GM_NEW_FLUSHED_ALARM_EVENT): - return("GM_NEW_FLUSHED_ALARM_EVENT"); - case(GM_NEW_SENT_TOKENS_EVENT): - return("GM_NEW_SENT_TOKENS_EVENT"); - case(GM_NEW_IGNORE_RECV_EVENT): - return("GM_NEW_IGNORE_RECV_EVENT"); - case(GM_NEW_ETHERNET_RECV_EVENT): - return("GM_NEW_ETHERNET_RECV_EVENT"); - default: - return("Unknown Recv event"); - /* _GM_PUT_NOTIFICATION_EVENT */ - /* GM_FREE_SEND_TOKEN_EVENT */ - /* GM_FREE_HIGH_SEND_TOKEN_EVENT */ - } -} - - -void -gmnal_yield(int delay) -{ - cfs_set_current_state(CFS_TASK_INTERRUPTIBLE); - cfs_schedule_timeout(delay); -} diff --git a/lnet/utils/.gitignore b/lnet/utils/.gitignore index b12aa50..2d3799a 100644 --- a/lnet/utils/.gitignore +++ b/lnet/utils/.gitignore @@ -6,7 +6,6 @@ /.deps /routerstat /wirecheck -/gmlndnid /lst /lstclient /.*.cmd diff --git a/lnet/utils/Makefile.am b/lnet/utils/Makefile.am index cfcabf5..5209901 100644 --- a/lnet/utils/Makefile.am +++ b/lnet/utils/Makefile.am @@ -67,19 +67,10 @@ libptlctl_a_SOURCES = portals.c debug.c if UTILS sbin_PROGRAMS += ptlctl routerstat wirecheck lst - -if BUILD_GMLND -sbin_PROGRAMS += gmlndnid -endif endif wirecheck_SOURCES = wirecheck.c -gmlndnid_SOURCES = gmlndnid.c -gmlndnid_CFLAGS = $(GMCPPFLAGS) -gmlndnid_LDFLAGS = -static -gmlndnid_LDADD = $(GMLIBS) -lgm - ptlctl_SOURCES = ptlctl.c ptlctl_LDADD = -L. -lptlctl $(LIBCFSUTIL) $(LIBREADLINE) $(LIBEFENCE) ptlctl_DEPENDENCIES = libptlctl.a diff --git a/lnet/utils/debug.c b/lnet/utils/debug.c index 75b8853..b9a498d 100644 --- a/lnet/utils/debug.c +++ b/lnet/utils/debug.c @@ -854,7 +854,6 @@ static struct mod_paths { {"libcfs", "libcfs/libcfs"}, {"lnet", "lnet/lnet"}, {"kciblnd", "lnet/klnds/ciblnd"}, - {"kgmlnd", "lnet/klnds/gmlnd"}, {"kmxlnd", "lnet/klnds/mxlnd"}, {"kiiblnd", "lnet/klnds/iiblnd"}, {"ko2iblnd", "lnet/klnds/o2iblnd"}, diff --git a/lnet/utils/gmlndnid.c b/lnet/utils/gmlndnid.c deleted file mode 100644 index 507aed8..0000000 --- a/lnet/utils/gmlndnid.c +++ /dev/null @@ -1,201 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. - * - * GPL HEADER END - */ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Use is subject to license terms. - * - * Copyright (c) 2003 Los Alamos National Laboratory (LANL) - */ -/* - * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -/* - * portals always uses unit 0 - * Can this be configurable? - */ -#define GM_UNIT 0 - -void -usage(char *prg, int h) -{ - fprintf(stderr, - "usage %s -h\n" - " %s [-l] [-n hostname] [-L] [hostnames]\n", prg); - - if (h) - printf("Print Myrinet Global network ids for specified hosts\n" - "-l print local host's ID\n" - "-n hostname print given host's ID\n" - "-L print Myringet local net ID too\n" - "[hostnames] print ids of given hosts (local if none)\n"); -} - -gm_status_t -print_gmid(char *name, int name_fieldlen, int show_local_id) -{ - struct gm_port *gm_port; - int gm_port_id; - gm_status_t gm_status; - unsigned int local_id; - unsigned int global_id; - - gm_status = gm_init(); - if (gm_status != GM_SUCCESS) { - fprintf(stderr, "gm_init: %s\n", gm_strerror(gm_status)); - return gm_status; - } - - gm_port_id = 2; - gm_status = gm_open(&gm_port, GM_UNIT, gm_port_id, "gmnalnid", - GM_API_VERSION); - if (gm_status != GM_SUCCESS) { - int num_ports = gm_num_ports(gm_port); - - /* Couldn't open port 2, try 4 ... num_ports */ - for (gm_port_id = 4; gm_port_id < num_ports; gm_port_id++) { - gm_status = gm_open(&gm_port, GM_UNIT, gm_port_id, - "gmnalnid", GM_API_VERSION); - if (gm_status == GM_SUCCESS) - break; - } - - if (gm_status != GM_SUCCESS) { - fprintf(stderr, "gm_open: %s\n",gm_strerror(gm_status)); - goto out_0; - } - } - - if (name == NULL) { - local_id = 1; - name = ""; - } else { - gm_status = gm_host_name_to_node_id_ex(gm_port, 1000000, name, - &local_id); - if (gm_status != GM_SUCCESS) { - fprintf(stderr, "gm_host_name_to_node_id_ex(%s): %s\n", - name, gm_strerror(gm_status)); - goto out_1; - } - } - - gm_status = gm_node_id_to_global_id(gm_port, local_id, &global_id) ; - if (gm_status != GM_SUCCESS) { - fprintf(stderr, "gm_node_id_to_global_id(%s:%d): %s\n", - name, local_id, gm_strerror(gm_status)); - goto out_1; - } - - if (name_fieldlen > 0) - printf ("%*s ", name_fieldlen, name); - - if (!show_local_id) - printf("0x%x\n", global_id); - else - printf("local 0x%x global 0x%x\n", local_id, global_id); - - out_1: - gm_close(gm_port); - out_0: - gm_finalize(); - - return gm_status; -} - -int -main (int argc, char **argv) -{ - int c; - gm_status_t gmrc; - int rc; - int max_namelen = 0; - int show_local_id = 0; - - while ((c = getopt(argc, argv, "n:lLh")) != -1) - switch(c) { - case 'h': - usage(argv[0], 1); - return 0; - - case 'L': - show_local_id = 1; - break; - - case 'n': - gmrc = print_gmid(optarg, 0, show_local_id); - return (gmrc == GM_SUCCESS) ? 0 : 1; - - case 'l': - gmrc = print_gmid(NULL, 0, show_local_id); - return (gmrc == GM_SUCCESS) ? 0 : 1; - - default: - usage(argv[0], 0); - return 2; - } - - if (optind == argc) { - gmrc = print_gmid(NULL, 0, show_local_id); - return (gmrc == GM_SUCCESS) ? 0 : 1; - } - - if (optind != argc - 1) - for (c = optind; c < argc; c++) - if (strlen(argv[c]) > max_namelen) - max_namelen = strlen(argv[c]); - - rc = 0; - - for (c = optind; c < argc; c++) { - gmrc = print_gmid(argv[c], max_namelen, show_local_id); - - if (gmrc != GM_SUCCESS) - rc = 1; - } - - return rc; -} diff --git a/lnet/utils/lnetunload b/lnet/utils/lnetunload index 8141178..fdc64ba 100755 --- a/lnet/utils/lnetunload +++ b/lnet/utils/lnetunload @@ -1,6 +1,6 @@ #!/bin/sh -lnds=$(echo k{sock,qsw,gm,{open,i,v,o2,c}ib,ra,ptl,mx}lnd) +lnds=$(echo k{sock,qsw,{open,i,v,o2,c}ib,ra,ptl,mx}lnd) do_rmmod() { mod=$1 diff --git a/lustre/scripts/lc_common b/lustre/scripts/lc_common index 2d67971..2cea2a8 100644 --- a/lustre/scripts/lc_common +++ b/lustre/scripts/lc_common @@ -310,10 +310,6 @@ nid2hostname() { # FIXME: Parse the /etc/elanhosts configuration file to # convert ElanID to hostname ;; - gm*) # Myrinet - # FIXME: Use /usr/sbin/gmlndnid to find the hostname of - # the specified GM Global node ID - ;; ptl*) # Portals # FIXME: Convert portal ID to hostname ;; @@ -360,7 +356,7 @@ nids2hostname() { [ "${nid}" != "${nid#*@*}" ] && nettype=${nid#*@} || nettype=tcp case "${nettype}" in - lo* | elan* | gm* | ptl*) ;; + lo* | elan* | ptl*) ;; *) # tcp, o2ib, cib, openib, iib, vib, ra host_name=$(nid2hostname ${nid}) if [ ${PIPESTATUS[0]} -ne 0 ]; then @@ -394,7 +390,7 @@ ip2hostname_single_node() { [ "${nid}" != "${nid#*@*}" ] && nettype=${nid#*@} || nettype=tcp case "${nettype}" in - lo* | elan* | gm* | ptl*) ;; + lo* | elan* | ptl*) ;; *) # tcp, o2ib, cib, openib, iib, vib, ra host_name=$(nid2hostname ${nid}) if [ ${PIPESTATUS[0]} -ne 0 ]; then diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 3027793..b72c512 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -9,7 +9,6 @@ set -e export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=false -export GMNALNID=${GMNALNID:-/usr/sbin/gmlndnid} export CATASTROPHE=${CATASTROPHE:-/proc/sys/lnet/catastrophe} export GSS=false export GSS_KRB5=false @@ -1398,13 +1397,6 @@ do_lmc() { exit 1 } -h2gm () { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - ID=`$PDSH $1 $GMNALNID -l | cut -d\ -f2` - echo $ID"@gm" - fi -} - h2name_or_ip() { if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else echo $1"@$2" diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 926a1af..8e08071 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -85,7 +85,7 @@ command_t cmdlist[] = { /* Network configuration commands */ {"===== network config =====", jt_noop, 0, "network config"}, {"--net", jt_opt_net, 0,"run after setting network to \n" - "usage: --net "}, + "usage: --net "}, {"network", jt_ptl_network, 0, "configure LNET" "usage: network up|down"}, {"net", jt_ptl_network, 0, "configure LNET"