X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Fautoconf%2Flustre-lnet.m4;h=734d8023becd8905958eefa7403e445477e12ef5;hp=41f42b5f3e84c2836d72c8c57a1e6deb8380b548;hb=605817db843fdd8e433751180e516d6497323956;hpb=356b11c36639b3470c197129a92f4cf26f61d87a diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index 41f42b5..734d802 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -1,33 +1,52 @@ # -# LP_CONFIG_ZEROCOPY +# LN_CONFIG_MAX_PAYLOAD # -# check if zerocopy is available/wanted +# configure maximum payload # -AC_DEFUN([LP_CONFIG_ZEROCOPY], -[AC_MSG_CHECKING([for zero-copy TCP support]) -AC_ARG_ENABLE([zerocopy], - AC_HELP_STRING([--disable-zerocopy], - [disable socknal zerocopy]), - [],[enable_zerocopy='yes']) -if test x$enable_zerocopy = xno ; then - AC_MSG_RESULT([no (by request)]) +AC_DEFUN([LN_CONFIG_MAX_PAYLOAD], +[AC_MSG_CHECKING([for non-default maximum LNET payload]) +AC_ARG_WITH([max-payload-mb], + AC_HELP_STRING([--with-max-payload-mb=MBytes], + [set maximum lnet payload in MBytes]), + [ + AC_MSG_RESULT([$with_max_payload_mb]) + LNET_MAX_PAYLOAD_MB=$with_max_payload_mb + LNET_MAX_PAYLOAD="(($with_max_payload_mb)<<20)" + ], [ + AC_MSG_RESULT([no]) + LNET_MAX_PAYLOAD="LNET_MTU" + ]) + AC_DEFINE_UNQUOTED(LNET_MAX_PAYLOAD, $LNET_MAX_PAYLOAD, + [Max LNET payload]) +]) + +# +# LN_CHECK_GCC_VERSION +# +# Check compiler version +# +AC_DEFUN([LN_CHECK_GCC_VERSION], +[AC_MSG_CHECKING([compiler version]) +PTL_CC_VERSION=`$CC --version | awk '/^gcc/{print $ 3}'` +PTL_MIN_CC_VERSION="3.2.2" +v2n() { + awk -F. '{printf "%d\n", (($ 1)*100+($ 2))*100+($ 3)}' +} +if test -z "$PTL_CC_VERSION" -o \ + `echo $PTL_CC_VERSION | v2n` -ge `echo $PTL_MIN_CC_VERSION | v2n`; then + AC_MSG_RESULT([ok]) else - ZCCD="`grep -c zccd $LINUX/include/linux/skbuff.h`" - if test "$ZCCD" != 0 ; then - AC_DEFINE(SOCKNAL_ZC, 1, [use zero-copy TCP]) - AC_MSG_RESULT(yes) - else - AC_MSG_RESULT([no (no kernel support)]) - fi + AC_MSG_RESULT([Buggy compiler found]) + AC_MSG_ERROR([Need gcc version >= $PTL_MIN_CC_VERSION]) fi ]) # -# LP_CONFIG_AFFINITY +# LN_CONFIG_AFFINITY # # check if cpu affinity is available/wanted # -AC_DEFUN([LP_CONFIG_AFFINITY], +AC_DEFUN([LN_CONFIG_AFFINITY], [AC_ARG_ENABLE([affinity], AC_HELP_STRING([--disable-affinity], [disable process/irq affinity]), @@ -41,10 +60,10 @@ else #include ],[ struct task_struct t; - #ifdef CPU_ARRAY_SIZE - cpumask_t m; - #else - unsigned long m; + #if HAVE_CPUMASK_T + cpumask_t m; + #else + unsigned long m; #endif set_cpus_allowed(&t, m); ],[ @@ -57,145 +76,830 @@ fi ]) # -# LP_CONFIG_QUADRICS +# LN_CONFIG_PORTALS +# +# configure support for Portals +# +AC_DEFUN([LN_CONFIG_PORTALS], +[AC_MSG_CHECKING([for portals]) +AC_ARG_WITH([portals], + AC_HELP_STRING([--with-portals=path], + [set path to portals]), + [ + case $with_portals in + no) ENABLEPORTALS=0 + ;; + *) PORTALS="${with_portals}" + ENABLEPORTALS=1 + ;; + esac + ], [ + ENABLEPORTALS=0 + ]) +PTLLNDCPPFLAGS="" +if test $ENABLEPORTALS -eq 0; then + AC_MSG_RESULT([no]) +elif test ! \( -f ${PORTALS}/include/portals/p30.h \); then + AC_MSG_RESULT([no]) + AC_MSG_ERROR([bad --with-portals path]) +else + AC_MSG_RESULT([$PORTALS]) + PTLLNDCPPFLAGS="-I${PORTALS}/include" +fi +AC_SUBST(PTLLNDCPPFLAGS) +]) + +# +# LN_CONFIG_BACKOFF +# +# check if tunable tcp backoff is available/wanted +# +AC_DEFUN([LN_CONFIG_BACKOFF], +[AC_MSG_CHECKING([for tunable backoff TCP support]) +AC_ARG_ENABLE([backoff], + AC_HELP_STRING([--disable-backoff], + [disable socknal tunable backoff]), + [],[enable_backoff='yes']) +if test x$enable_backoff = xno ; then + AC_MSG_RESULT([no (by request)]) +else + BOCD="`grep -c TCP_BACKOFF $LINUX/include/linux/tcp.h`" + if test "$BOCD" != 0 ; then + AC_DEFINE(SOCKNAL_BACKOFF, 1, [use tunable backoff TCP]) + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT([no (no kernel support)]) + fi +fi +]) + + +# +# LN_CONFIG_PTLLND +# +# configure support for Portals LND +# +AC_DEFUN([LN_CONFIG_PTLLND], +[ +if test -z "$ENABLEPORTALS"; then + LN_CONFIG_PORTALS +fi + +AC_MSG_CHECKING([whether to build the kernel portals LND]) + +PTLLND="" +if test $ENABLEPORTALS -ne 0; then + AC_MSG_RESULT([yes]) + PTLLND="ptllnd" +else + AC_MSG_RESULT([no]) +fi +AC_SUBST(PTLLND) +]) + +# +# LN_CONFIG_UPTLLND +# +# configure support for Portals LND +# +AC_DEFUN([LN_CONFIG_UPTLLND], +[ +if test -z "$ENABLEPORTALS"; then + LN_CONFIG_PORTALS +fi + +AC_MSG_CHECKING([whether to build the userspace portals LND]) + +UPTLLND="" +if test $ENABLEPORTALS -ne 0; then + AC_MSG_RESULT([yes]) + UPTLLND="ptllnd" +else + AC_MSG_RESULT([no]) +fi +AC_SUBST(UPTLLND) +]) + +# +# LN_CONFIG_USOCKLND +# +# configure support for userspace TCP/IP LND +# +AC_DEFUN([LN_CONFIG_USOCKLND], +[AC_MSG_CHECKING([whether to build usocklnd]) +AC_ARG_ENABLE([usocklnd], + AC_HELP_STRING([--disable-usocklnd], + [disable usocklnd]), + [],[enable_usocklnd='yes']) + +if test x$enable_usocklnd = xyes ; then + if test "$ENABLE_LIBPTHREAD" = "yes" ; then + AC_MSG_RESULT([yes]) + USOCKLND="usocklnd" + else + AC_MSG_RESULT([no (libpthread not present or disabled)]) + USOCKLND="" + fi +else + AC_MSG_RESULT([no (disabled explicitly)]) + USOCKLND="" +fi +AC_SUBST(USOCKLND) +]) + +# +# LN_CONFIG_QUADRICS # # check if quadrics support is in this kernel # -AC_DEFUN([LP_CONFIG_QUADRICS], -[AC_MSG_CHECKING([if quadrics kernel headers are present]) -if test -d $LINUX/drivers/net/qsnet ; then +AC_DEFUN([LN_CONFIG_QUADRICS], +[AC_MSG_CHECKING([for QsNet sources]) +AC_ARG_WITH([qsnet], + AC_HELP_STRING([--with-qsnet=path], + [set path to qsnet source (default=$LINUX)]), + [QSNET=$with_qsnet], + [QSNET=$LINUX]) +AC_MSG_RESULT([$QSNET]) + +AC_MSG_CHECKING([if quadrics kernel headers are present]) +if test -d $QSNET/drivers/net/qsnet ; then AC_MSG_RESULT([yes]) - QSWNAL="qswnal" + QSWLND="qswlnd" AC_MSG_CHECKING([for multirail EKC]) - if test -f $LINUX/include/elan/epcomms.h; then + if test -f $QSNET/include/elan/epcomms.h; then AC_MSG_RESULT([supported]) - QSWCPPFLAGS="-DMULTIRAIL_EKC=1" + QSWCPPFLAGS="-I$QSNET/include -DMULTIRAIL_EKC=1" else AC_MSG_RESULT([not supported]) - if test -d $LINUX/drivers/net/qsnet/include; then - QSWCPPFLAGS="-I$LINUX/drivers/net/qsnet/include" - else - QSWCPPFLAGS="-I$LINUX/include/linux" - fi + AC_MSG_ERROR([Need multirail EKC]) + fi + + if test x$QSNET = x$LINUX ; then + LB_LINUX_CONFIG([QSNET],[],[ + LB_LINUX_CONFIG([QSNET_MODULE],[],[ + AC_MSG_WARN([QSNET is not enabled in this kernel; not building qswlnd.]) + QSWLND="" + QSWCPPFLAGS="" + ]) + ]) fi else AC_MSG_RESULT([no]) - QSWNAL="" + QSWLND="" QSWCPPFLAGS="" fi AC_SUBST(QSWCPPFLAGS) -AC_SUBST(QSWNAL) +AC_SUBST(QSWLND) ]) # -# LP_CONFIG_GM +# LN_CONFIG_GM # -# check if infiniband support is available +# check if GM support is available # -AC_DEFUN([LP_CONFIG_GM], -[AC_MSG_CHECKING([if gm support was requested]) +AC_DEFUN([LN_CONFIG_GM],[ +AC_MSG_CHECKING([whether to enable GM support]) AC_ARG_WITH([gm], - AC_HELP_STRING([--with-gm=path], - [build gmnal against path]), + AC_HELP_STRING([--with-gm=path-to-gm-source-tree], + [build gmlnd against path]), [ - case $with_gm in - yes) - AC_MSG_RESULT([yes]) - GMCPPFLAGS="-I/usr/local/gm/include" - GMNAL="gmnal" - ;; - no) - AC_MSG_RESULT([no]) - GMCPPFLAGS="" - GMNAL="" - ;; - *) - AC_MSG_RESULT([yes]) - GMCPPFLAGS="-I$with_gm/include -I$with_gm/drivers -I$with_gm/drivers/linux/gm" - GMNAL="gmnal" - ;; + case $with_gm in + no) ENABLE_GM=0 + ;; + *) ENABLE_GM=1 + GM_SRC="$with_gm" + ;; + esac + ],[ + ENABLE_GM=0 + ]) +AC_ARG_WITH([gm-install], + AC_HELP_STRING([--with-gm-install=path-to-gm-install-tree], + [say where GM has been installed]), + [ + GM_INSTALL=$with_gm_install + ],[ + GM_INSTALL="/opt/gm" + ]) +if test $ENABLE_GM -eq 0; then + AC_MSG_RESULT([no]) +else + AC_MSG_RESULT([yes]) + + GMLND="gmlnd" + GMCPPFLAGS="-I$GM_SRC/include -I$GM_SRC/drivers -I$GM_SRC/drivers/linux/gm" + + if test -f $GM_INSTALL/lib/libgm.a -o \ + -f $GM_INSTALL/lib64/libgm.a; then + GMLIBS="-L$GM_INSTALL/lib -L$GM_INSTALL/lib64" + else + AC_MSG_ERROR([Cant find GM libraries under $GM_INSTALL]) + fi + + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$GMCPPFLAGS -DGM_KERNEL $EXTRA_KCFLAGS" + + AC_MSG_CHECKING([that code using GM compiles with given path]) + LB_LINUX_TRY_COMPILE([ + #define GM_STRONG_TYPES 1 + #ifdef VERSION + #undef VERSION + #endif + #include "gm.h" + #include "gm_internal.h" + ],[ + struct gm_port *port = NULL; + gm_recv_event_t *rxevent = gm_blocking_receive_no_spin(port); + return 0; + ],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([Bad --with-gm path]) + ]) + + AC_MSG_CHECKING([that GM has gm_register_memory_ex_phys()]) + LB_LINUX_TRY_COMPILE([ + #define GM_STRONG_TYPES 1 + #ifdef VERSION + #undef VERSION + #endif + #include "gm.h" + #include "gm_internal.h" + ],[ + gm_status_t gmrc; + struct gm_port *port = NULL; + gm_u64_t phys = 0; + gm_up_t pvma = 0; + + gmrc = gm_register_memory_ex_phys(port, phys, 100, pvma); + return 0; + ],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no. +Please patch the GM sources as follows... + cd $GM_SRC + patch -p0 < $PWD/lnet/klnds/gmlnd/gm-reg-phys.patch +...then rebuild and re-install them]) + AC_MSG_ERROR([Can't build GM without gm_register_memory_ex_phys()]) + ]) + + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi +AC_SUBST(GMCPPFLAGS) +AC_SUBST(GMLIBS) +AC_SUBST(GMLND) +]) + + +# +# LN_CONFIG_MX +# +AC_DEFUN([LN_CONFIG_MX], +[AC_MSG_CHECKING([whether to enable Myrinet MX support]) +# set default +MXPATH="/opt/mx" +AC_ARG_WITH([mx], + AC_HELP_STRING([--with-mx=path], + [build mxlnd against path]), + [ + case $with_mx in + yes) ENABLEMX=2 + ;; + no) ENABLEMX=0 + ;; + *) MXPATH=$with_mx + ENABLEMX=3 + ;; + esac + ],[ + ENABLEMX=1 + ]) +if test $ENABLEMX -eq 0; then + AC_MSG_RESULT([disabled]) +elif test ! \( -f ${MXPATH}/include/myriexpress.h -a \ + -f ${MXPATH}/include/mx_kernel_api.h -a \ + -f ${MXPATH}/include/mx_pin.h \); then + AC_MSG_RESULT([no]) + case $ENABLEMX in + 1) ;; + 2) AC_MSG_ERROR([Myrinet MX kernel headers not present]);; + 3) AC_MSG_ERROR([bad --with-mx path]);; + *) AC_MSG_ERROR([internal error]);; + esac +else + MXCPPFLAGS="-I$MXPATH/include" + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $MXCPPFLAGS" + MXLIBS="-L$MXPATH/lib" + LB_LINUX_TRY_COMPILE([ + #define MX_KERNEL 1 + #include + #include + ],[ + mx_endpoint_t end; + mx_status_t status; + mx_request_t request; + int result; + + mx_init(); + mx_open_endpoint(MX_ANY_NIC, MX_ANY_ENDPOINT, 0, NULL, 0, &end); + mx_register_unexp_handler(end, (mx_unexp_handler_t) NULL, NULL); + mx_wait_any(end, MX_INFINITE, 0LL, 0LL, &status, &result); + mx_iconnect(end, 0LL, 0, 0, 0, NULL, &request); + return 0; + ],[ + AC_MSG_RESULT([yes]) + MXLND="mxlnd" + ],[ + AC_MSG_RESULT([no]) + case $ENABLEMX in + 1) ;; + 2) AC_MSG_ERROR([can't compile with Myrinet MX kernel headers]);; + 3) AC_MSG_ERROR([can't compile with Myrinet MX headers under $MXPATH]);; + *) AC_MSG_ERROR([internal error]);; + esac + MXLND="" + MXCPPFLAGS="" + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi +AC_SUBST(MXCPPFLAGS) +AC_SUBST(MXLIBS) +AC_SUBST(MXLND) +]) + + + +# +# LN_CONFIG_O2IB +# +AC_DEFUN([LN_CONFIG_O2IB],[ +AC_MSG_CHECKING([whether to enable OpenIB gen2 support]) +# set default +AC_ARG_WITH([o2ib], + AC_HELP_STRING([--with-o2ib=path], + [build o2iblnd against path]), + [ + case $with_o2ib in + yes) O2IBPATHS="$LINUX $LINUX/drivers/infiniband" + ENABLEO2IB=2 + ;; + no) ENABLEO2IB=0 + ;; + *) O2IBPATHS=$with_o2ib + ENABLEO2IB=3 + ;; esac ],[ - AC_MSG_RESULT([no]) - GMCPPFLAGS="" - GMNAL="" + O2IBPATHS="$LINUX $LINUX/drivers/infiniband" + ENABLEO2IB=1 ]) -AC_SUBST(GMCPPFLAGS) -AC_SUBST(GMNAL) +if test $ENABLEO2IB -eq 0; then + AC_MSG_RESULT([disabled]) +else + o2ib_found=false + + for O2IBPATH in $O2IBPATHS; do + if test \( -f ${O2IBPATH}/include/rdma/rdma_cm.h -a \ + -f ${O2IBPATH}/include/rdma/ib_cm.h -a \ + -f ${O2IBPATH}/include/rdma/ib_verbs.h -a \ + -f ${O2IBPATH}/include/rdma/ib_fmr_pool.h \); then + o2ib_found=true + break + fi + done + + if ! $o2ib_found; then + AC_MSG_RESULT([no]) + case $ENABLEO2IB in + 1) ;; + 2) AC_MSG_ERROR([kernel OpenIB gen2 headers not present]);; + 3) AC_MSG_ERROR([bad --with-o2ib path]);; + *) AC_MSG_ERROR([internal error]);; + esac + else + O2IBCPPFLAGS="-I$O2IBPATH/include" + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $O2IBCPPFLAGS" + EXTRA_LNET_INCLUDE="$O2IBCPPFLAGS $EXTRA_LNET_INCLUDE" + LB_LINUX_TRY_COMPILE([ + #include + #include + #if !HAVE_GFP_T + typedef int gfp_t; + #endif + #include + #include + #include + #include + ],[ + struct rdma_cm_id *cm_id; + struct rdma_conn_param conn_param; + struct ib_device_attr device_attr; + struct ib_qp_attr qp_attr; + struct ib_pool_fmr pool_fmr; + enum ib_cm_rej_reason rej_reason; + + cm_id = rdma_create_id(NULL, NULL, RDMA_PS_TCP); + return PTR_ERR(cm_id); + ],[ + AC_MSG_RESULT([yes]) + O2IBLND="o2iblnd" + ],[ + AC_MSG_RESULT([no]) + case $ENABLEO2IB in + 1) ;; + 2) AC_MSG_ERROR([can't compile with kernel OpenIB gen2 headers]);; + 3) AC_MSG_ERROR([can't compile with OpenIB gen2 headers under $O2IBPATH]);; + *) AC_MSG_ERROR([internal error]);; + esac + O2IBLND="" + O2IBCPPFLAGS="" + ]) + # we know at this point that the found OFED source is good + O2IB_SYMVER="" + if test $ENABLEO2IB -eq 3 ; then + # OFED default rpm not handle sles10 Modules.symvers name + for name in Module.symvers Modules.symvers; do + if test -f $O2IBPATH/$name; then + O2IB_SYMVER=$name; + break; + fi + done + if test -n $O2IB_SYMVER ; then + AC_MSG_NOTICE([adding $O2IBPATH/$O2IB_SYMVER to $PWD/$SYMVERFILE]) + # strip out the existing symbols versions first + egrep -v $(echo $(awk '{ print $2 }' $O2IBPATH/$O2IB_SYMVER) | tr ' ' '|') $PWD/$SYMVERFILE > $PWD/$SYMVERFILE.old + cat $PWD/$SYMVERFILE.old $O2IBPATH/$O2IB_SYMVER > $PWD/$SYMVERFILE + else + AC_MSG_ERROR([an external source tree was specified for o2iblnd however I could not find a $O2IBPATH/Module.symvers there]) + fi + fi + + LB_LINUX_TRY_COMPILE([ + #include + #include + #if !HAVE_GFP_T + typedef int gfp_t; + #endif + #include + ],[ + ib_dma_map_single(NULL, NULL, 0, 0); + return 0; + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_OFED_IB_DMA_MAP, 1, + [ib_dma_map_single defined]) + ],[ + AC_MSG_RESULT(NO) + ]) + + LB_LINUX_TRY_COMPILE([ + #include + #include + #if !HAVE_GFP_T + typedef int gfp_t; + #endif + #include + ],[ + ib_create_cq(NULL, NULL, NULL, NULL, 0, 0); + return 0; + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_OFED_IB_COMP_VECTOR, 1, + [has completion vector]) + ],[ + AC_MSG_RESULT(NO) + ]) + + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" + fi +fi + +AC_SUBST(EXTRA_LNET_INCLUDE) +AC_SUBST(O2IBCPPFLAGS) +AC_SUBST(O2IBLND) ]) # -# LP_CONFIG_OPENIB +# LN_CONFIG_OPENIB # # check for OpenIB in the kernel -AC_DEFUN([LP_CONFIG_OPENIB], -[AC_MSG_CHECKING([if OpenIB kernel headers are present]) -OPENIBCPPFLAGS="-I$LINUX/drivers/infiniband/include -DIN_TREE_BUILD" -EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" -EXTRA_KCFLAGS="$EXTRA_KCFLAGS $OPENIBCPPFLAGS" -LB_LINUX_TRY_COMPILE( +AC_DEFUN([LN_CONFIG_OPENIB],[ +AC_MSG_CHECKING([whether to enable OpenIB support]) +# set default +OPENIBPATH="$LINUX/drivers/infiniband" +AC_ARG_WITH([openib], + AC_HELP_STRING([--with-openib=path], + [build openiblnd against path]), [ + case $with_openib in + yes) ENABLEOPENIB=2 + ;; + no) ENABLEOPENIB=0 + ;; + *) OPENIBPATH="$with_openib" + ENABLEOPENIB=3 + ;; + esac + ],[ + ENABLEOPENIB=1 + ]) +if test $ENABLEOPENIB -eq 0; then + AC_MSG_RESULT([disabled]) +elif test ! \( -f ${OPENIBPATH}/include/ts_ib_core.h -a \ + -f ${OPENIBPATH}/include/ts_ib_cm.h -a \ + -f ${OPENIBPATH}/include/ts_ib_sa_client.h \); then + AC_MSG_RESULT([no]) + case $ENABLEOPENIB in + 1) ;; + 2) AC_MSG_ERROR([kernel OpenIB headers not present]);; + 3) AC_MSG_ERROR([bad --with-openib path]);; + *) AC_MSG_ERROR([internal error]);; + esac +else + case $ENABLEOPENIB in + 1|2) OPENIBCPPFLAGS="-I$OPENIBPATH/include -DIN_TREE_BUILD";; + 3) OPENIBCPPFLAGS="-I$OPENIBPATH/include";; + *) AC_MSG_RESULT([no]) + AC_MSG_ERROR([internal error]);; + esac + OPENIBCPPFLAGS="$OPENIBCPPFLAGS -DIB_NTXRXPARAMS=4" + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $OPENIBCPPFLAGS" + LB_LINUX_TRY_COMPILE([ #include + #include + #include ],[ - struct ib_device_properties props; + struct ib_device_properties dev_props; + struct ib_cm_active_param cm_active_params; + tTS_IB_CLIENT_QUERY_TID tid; + int enum1 = IB_QP_ATTRIBUTE_STATE; + int enum2 = IB_ACCESS_LOCAL_WRITE; + int enum3 = IB_CQ_CALLBACK_INTERRUPT; + int enum4 = IB_CQ_PROVIDER_REARM; return 0; ],[ AC_MSG_RESULT([yes]) - OPENIBNAL="openibnal" + OPENIBLND="openiblnd" ],[ AC_MSG_RESULT([no]) - OPENIBNAL="" + case $ENABLEOPENIB in + 1) ;; + 2) AC_MSG_ERROR([can't compile with kernel OpenIB headers]);; + 3) AC_MSG_ERROR([can't compile with OpenIB headers under $OPENIBPATH]);; + *) AC_MSG_ERROR([internal error]);; + esac + OPENIBLND="" OPENIBCPPFLAGS="" ]) -EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi AC_SUBST(OPENIBCPPFLAGS) -AC_SUBST(OPENIBNAL) +AC_SUBST(OPENIBLND) +]) + +# +# LN_CONFIG_CIBLND +# +AC_DEFUN([LN_CONFIG_CIB],[ +AC_MSG_CHECKING([whether to enable Cisco/TopSpin IB support]) +# set default +CIBPATH="" +CIBLND="" +AC_ARG_WITH([cib], + AC_HELP_STRING([--with-cib=path], + [build ciblnd against path]), + [ + case $with_cib in + no) AC_MSG_RESULT([no]);; + *) CIBPATH="$with_cib" + if test -d "$CIBPATH"; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + AC_MSG_ERROR([No directory $CIBPATH]) + fi;; + esac + ],[ + AC_MSG_RESULT([no]) + ]) +if test -n "$CIBPATH"; then + CIBCPPFLAGS="-I${CIBPATH}/ib/ts_api_ng/include -I${CIBPATH}/all/kernel_services/include -DUSING_TSAPI" + CIBCPPFLAGS="$CIBCPPFLAGS -DIB_NTXRXPARAMS=3" + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $CIBCPPFLAGS" + LB_LINUX_TRY_COMPILE([ + #include + #include + #include + ],[ + struct ib_device_properties dev_props; + struct ib_cm_active_param cm_active_params; + tTS_IB_CLIENT_QUERY_TID tid; + int enum1 = TS_IB_QP_ATTRIBUTE_STATE; + int enum2 = TS_IB_ACCESS_LOCAL_WRITE; + int enum3 = TS_IB_CQ_CALLBACK_INTERRUPT; + int enum4 = TS_IB_CQ_PROVIDER_REARM; + return 0; + ],[ + CIBLND="ciblnd" + ],[ + AC_MSG_ERROR([can't compile ciblnd with given path]) + CIBCPPFLAGS="" + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi +AC_SUBST(CIBCPPFLAGS) +AC_SUBST(CIBLND) ]) # -# LP_CONFIG_IIB +# LN_CONFIG_IIB # # check for infinicon infiniband support # -AC_DEFUN([LP_CONFIG_IIB], -[AC_MSG_CHECKING([if Infinicon IB kernel headers are present]) -# for how the only infinicon ib build has headers in /usr/include/iba -IIBCPPFLAGS="-I/usr/include -DIN_TREE_BUILD" -EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" -EXTRA_KCFLAGS="$EXTRA_KCFLAGS $IIBCPPFLAGS" -LB_LINUX_TRY_COMPILE([ - #include -],[ - IBT_INTERFACE_UNION interfaces; - FSTATUS rc; +AC_DEFUN([LN_CONFIG_IIB],[ +AC_MSG_CHECKING([whether to enable Infinicon support]) +# set default +IIBPATH="/usr/include" +AC_ARG_WITH([iib], + AC_HELP_STRING([--with-iib=path], + [build iiblnd against path]), + [ + case $with_iib in + yes) ENABLEIIB=2 + ;; + no) ENABLEIIB=0 + ;; + *) IIBPATH="${with_iib}/include" + ENABLEIIB=3 + ;; + esac + ],[ + ENABLEIIB=1 + ]) +if test $ENABLEIIB -eq 0; then + AC_MSG_RESULT([disabled]) +elif test ! \( -f ${IIBPATH}/linux/iba/ibt.h \); then + AC_MSG_RESULT([no]) + case $ENABLEIIB in + 1) ;; + 2) AC_MSG_ERROR([default Infinicon headers not present]);; + 3) AC_MSG_ERROR([bad --with-iib path]);; + *) AC_MSG_ERROR([internal error]);; + esac +else + IIBCPPFLAGS="-I$IIBPATH" + if test $IIBPATH != "/usr/include"; then + # we need /usr/include come what may + IIBCPPFLAGS="$IIBCPPFLAGS -I/usr/include" + fi + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $IIBCPPFLAGS" + LB_LINUX_TRY_COMPILE([ + #include + ],[ + IBT_INTERFACE_UNION interfaces; + FSTATUS rc; - rc = IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2, - &interfaces); + rc = IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2, + &interfaces); - return rc == FSUCCESS ? 0 : 1; -],[ - AC_MSG_RESULT([yes]) - IIBNAL="iibnal" -],[ - AC_MSG_RESULT([no]) - IIBNAL="" - IIBCPPFLAGS="" -]) -EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" + return rc == FSUCCESS ? 0 : 1; + ],[ + AC_MSG_RESULT([yes]) + IIBLND="iiblnd" + ],[ + AC_MSG_RESULT([no]) + case $ENABLEIIB in + 1) ;; + 2) AC_MSG_ERROR([can't compile with default Infinicon headers]);; + 3) AC_MSG_ERROR([can't compile with Infinicon headers under $IIBPATH]);; + *) AC_MSG_ERROR([internal error]);; + esac + IIBLND="" + IIBCPPFLAGS="" + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi AC_SUBST(IIBCPPFLAGS) -AC_SUBST(IIBNAL) +AC_SUBST(IIBLND) +]) + +# +# LN_CONFIG_VIB +# +# check for Voltaire infiniband support +# +AC_DEFUN([LN_CONFIG_VIB], +[AC_MSG_CHECKING([whether to enable Voltaire IB support]) +VIBPATH="" +AC_ARG_WITH([vib], + AC_HELP_STRING([--with-vib=path], + [build viblnd against path]), + [ + case $with_vib in + no) AC_MSG_RESULT([no]);; + *) VIBPATH="${with_vib}/src/nvigor/ib-code" + if test -d "$with_vib" -a -d "$VIBPATH"; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + AC_MSG_ERROR([No directory $VIBPATH]) + fi;; + esac + ],[ + AC_MSG_RESULT([no]) + ]) +if test -z "$VIBPATH"; then + VIBLND="" +else + VIBCPPFLAGS="-I${VIBPATH}/include -I${VIBPATH}/cm" + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $VIBCPPFLAGS" + LB_LINUX_TRY_COMPILE([ + #include + #include + #ifdef __BIG_ENDIAN + # define CPU_BE 1 + # define CPU_LE 0 + #endif + #ifdef __LITTLE_ENDIAN + # define CPU_BE 0 + # define CPU_LE 1 + #endif + #include + #include + #include + ],[ + vv_hca_h_t kib_hca; + vv_return_t vvrc; + cm_cep_handle_t cep; + ibat_arp_data_t arp_data; + ibat_stat_t ibatrc; + + vvrc = vv_hca_open("ANY_HCA", NULL, &kib_hca); + cep = cm_create_cep(cm_cep_transp_rc); + ibatrc = ibat_get_ib_data((uint32_t)0, (uint32_t)0, + ibat_paths_primary, &arp_data, + (ibat_get_ib_data_reply_fn_t)NULL, + NULL, 0); + return 0; + ],[ + VIBLND="viblnd" + ],[ + AC_MSG_ERROR([can't compile viblnd with given path]) + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi +if test -n "$VIBLND"; then + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $VIBCPPFLAGS" + AC_MSG_CHECKING([if Voltaire still uses void * sg addresses]) + LB_LINUX_TRY_COMPILE([ + #include + #include + #ifdef __BIG_ENDIAN + # define CPU_BE 1 + # define CPU_LE 0 + #endif + #ifdef __LITTLE_ENDIAN + # define CPU_BE 0 + # define CPU_LE 1 + #endif + #include + #include + #include + ],[ + vv_scatgat_t sg; + + return &sg.v_address[3] == NULL; + ],[ + AC_MSG_RESULT([yes]) + VIBCPPFLAGS="$VIBCPPFLAGS -DIBNAL_VOIDSTAR_SGADDR=1" + ],[ + AC_MSG_RESULT([no]) + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi +AC_SUBST(VIBCPPFLAGS) +AC_SUBST(VIBLND) ]) # -# LP_CONFIG_RANAL +# LN_CONFIG_RALND # -# check whether to use the RapidArray nal +# check whether to use the RapidArray lnd # -AC_DEFUN([LP_CONFIG_RANAL], +AC_DEFUN([LN_CONFIG_RALND], [#### Rapid Array AC_MSG_CHECKING([if RapidArray kernel headers are present]) # placeholder -RACPPFLAGS="-I/tmp" +RACPPFLAGS="-I${LINUX}/drivers/xd1/include" EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" EXTRA_KCFLAGS="$EXTRA_KCFLAGS $RACPPFLAGS" LB_LINUX_TRY_COMPILE([ @@ -210,161 +914,69 @@ LB_LINUX_TRY_COMPILE([ return rc == RAP_SUCCESS ? 0 : 1; ],[ AC_MSG_RESULT([yes]) - RANAL="ranal" + RALND="ralnd" ],[ AC_MSG_RESULT([no]) - RANAL="" + RALND="" RACPPFLAGS="" ]) EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" AC_SUBST(RACPPFLAGS) -AC_SUBST(RANAL) +AC_SUBST(RALND) ]) -# -# LP_STRUCT_PAGE_LIST -# -# 2.6.4 no longer has page->list -# -AC_DEFUN([LP_STRUCT_PAGE_LIST], -[AC_MSG_CHECKING([if struct page has a list field]) -LB_LINUX_TRY_COMPILE([ - #include -],[ - struct page page; - &page.list; -],[ - AC_MSG_RESULT([yes]) - AC_DEFINE(HAVE_PAGE_LIST, 1, [struct page has a list field]) -],[ - AC_MSG_RESULT([no]) -]) -]) - -# -# LP_STRUCT_SIGHAND -# -# red hat 2.4 adds sighand to struct task_struct -# -AC_DEFUN([LP_STRUCT_SIGHAND], -[AC_MSG_CHECKING([if task_struct has a sighand field]) -LB_LINUX_TRY_COMPILE([ - #include -],[ - struct task_struct p; - p.sighand = NULL; -],[ - AC_DEFINE(CONFIG_RH_2_4_20, 1, [this kernel contains Red Hat 2.4.20 patches]) - AC_MSG_RESULT([yes]) -],[ - AC_MSG_RESULT([no]) -]) -]) - -# -# LP_FUNC_CPU_ONLINE -# -# cpu_online is different in rh 2.4, vanilla 2.4, and 2.6 -# -AC_DEFUN([LP_FUNC_CPU_ONLINE], -[AC_MSG_CHECKING([if kernel defines cpu_online()]) -LB_LINUX_TRY_COMPILE([ - #include -],[ - cpu_online(0); -],[ - AC_MSG_RESULT([yes]) - AC_DEFINE(HAVE_CPU_ONLINE, 1, [cpu_online found]) -],[ - AC_MSG_RESULT([no]) -]) -]) -# -# LP_TYPE_CPUMASK_T -# -# same goes for cpumask_t -# -AC_DEFUN([LP_TYPE_CPUMASK_T], -[AC_MSG_CHECKING([if kernel defines cpumask_t]) -LB_LINUX_TRY_COMPILE([ - #include -],[ - return sizeof (cpumask_t); -],[ - AC_MSG_RESULT([yes]) - AC_DEFINE(HAVE_CPUMASK_T, 1, [cpumask_t found]) -],[ - AC_MSG_RESULT([no]) -]) -]) # -# LP_FUNC_SHOW_TASK +# LN_PROG_LINUX # -# we export show_task(), but not all kernels have it (yet) +# LNet linux kernel checks # -AC_DEFUN([LP_FUNC_SHOW_TASK], -[AC_MSG_CHECKING([if kernel exports show_task]) -have_show_task=0 -for file in ksyms sched ; do - if grep -q "EXPORT_SYMBOL(show_task)" \ - "$LINUX/kernel/$file.c" 2>/dev/null ; then - have_show_task=1 - break - fi -done -if test x$have_show_task = x1 ; then - AC_DEFINE(HAVE_SHOW_TASK, 1, [show_task is exported]) - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi +AC_DEFUN([LN_PROG_LINUX], +[ +LN_CONFIG_AFFINITY +LN_CONFIG_BACKOFF +LN_CONFIG_QUADRICS +LN_CONFIG_GM +LN_CONFIG_OPENIB +LN_CONFIG_CIB +LN_CONFIG_VIB +LN_CONFIG_IIB +LN_CONFIG_O2IB +LN_CONFIG_RALND +LN_CONFIG_PTLLND +LN_CONFIG_MX ]) # -# LP_PROG_LINUX +# LN_PROG_DARWIN # -# Portals linux kernel checks +# Darwin checks # -AC_DEFUN([LP_PROG_LINUX], -[LP_CONFIG_ZEROCOPY -LP_CONFIG_AFFINITY -LP_CONFIG_QUADRICS -LP_CONFIG_GM -if test $linux25 = 'no' ; then - LP_CONFIG_OPENIB -fi -LP_CONFIG_IIB -LP_CONFIG_RANAL - -LP_STRUCT_PAGE_LIST -LP_STRUCT_SIGHAND -LP_FUNC_CPU_ONLINE -LP_TYPE_CPUMASK_T -LP_FUNC_SHOW_TASK +AC_DEFUN([LN_PROG_DARWIN], +[LB_DARWIN_CHECK_FUNCS([get_preemption_level]) ]) # -# LP_PATH_DEFAULTS +# LN_PATH_DEFAULTS # # default paths for installed files # -AC_DEFUN([LP_PATH_DEFAULTS], +AC_DEFUN([LN_PATH_DEFAULTS], [ ]) # -# LP_CONFIGURE +# LN_CONFIGURE # # other configure checks # -AC_DEFUN([LP_CONFIGURE], -[# portals/utils/portals.c -AC_CHECK_HEADERS([netdb.h netinet/tcp.h asm/types.h]) +AC_DEFUN([LN_CONFIGURE], +[# lnet/utils/portals.c +AC_CHECK_HEADERS([netdb.h netinet/tcp.h asm/types.h endian.h sys/ioctl.h]) AC_CHECK_FUNCS([gethostbyname socket connect]) -# portals/utils/debug.c +# lnet/utils/debug.c AC_CHECK_HEADERS([linux/version.h]) AC_CHECK_TYPE([spinlock_t], @@ -372,38 +984,39 @@ AC_CHECK_TYPE([spinlock_t], [], [#include ]) +# lnet/utils/wirecheck.c +AC_CHECK_FUNCS([strnlen]) + # -------- Check for required packages -------------- -# this doesn't seem to work on older autoconf -# AC_CHECK_LIB(readline, readline,,) -AC_MSG_CHECKING([for readline support]) +# +# LC_CONFIG_READLINE +# +# Build with readline +# +AC_MSG_CHECKING([whether to enable readline support]) AC_ARG_ENABLE(readline, - AC_HELP_STRING([--disable-readline], - [do not use readline library]), - [],[enable_readline='yes']) -AC_MSG_RESULT([$enable_readline]) + AC_HELP_STRING([--disable-readline], + [disable readline support]), + [],[enable_readline='yes']) +AC_MSG_RESULT([$enable_readline]) + +# -------- check for readline if enabled ---- if test x$enable_readline = xyes ; then + LIBS_save="$LIBS" + LIBS="-lncurses $LIBS" + AC_CHECK_LIB([readline],[readline],[ LIBREADLINE="-lreadline -lncurses" AC_DEFINE(HAVE_LIBREADLINE, 1, [readline library is available]) -else + ],[ + LIBREADLINE="" + ]) + LIBS="$LIBS_save" +else LIBREADLINE="" fi AC_SUBST(LIBREADLINE) -AC_MSG_CHECKING([if efence debugging support is requested]) -AC_ARG_ENABLE(efence, - AC_HELP_STRING([--enable-efence], - [use efence library]), - [],[enable_efence='no']) -AC_MSG_RESULT([$enable_efence]) -if test "$enable_efence" = "yes" ; then - LIBEFENCE="-lefence" - AC_DEFINE(HAVE_LIBEFENCE, 1, [libefence support is requested]) -else - LIBEFENCE="" -fi -AC_SUBST(LIBEFENCE) - # -------- enable acceptor libwrap (TCP wrappers) support? ------- AC_MSG_CHECKING([if libwrap support is requested]) AC_ARG_ENABLE([libwrap], @@ -445,78 +1058,99 @@ if test x$enable_urandom != xno ; then AC_DEFINE([LIBLUSTRE_USE_URANDOM], 1, [use /dev/urandom for random data]) fi -# -------- check for -lcap and -lpthread ---- +# -------- check for -lcap support ---- if test x$enable_liblustre = xyes ; then AC_CHECK_LIB([cap], [cap_get_proc], [ CAP_LIBS="-lcap" AC_DEFINE([HAVE_LIBCAP], 1, [use libcap]) ], - [CAP_LIBS=""]) - AC_SUBST(CAP_LIBS) - AC_CHECK_LIB([pthread], [pthread_create], [ - PTHREAD_LIBS="-lpthread" - AC_DEFINE([HAVE_LIBPTHREAD], 1, [use libpthread]) - ], - [PTHREAD_LIBS=""]) - AC_SUBST(PTHREAD_LIBS) + CAP_LIBS="" + ]) + AC_SUBST(CAP_LIBS) + fi + +LN_CONFIG_MAX_PAYLOAD +LN_CONFIG_UPTLLND +LN_CONFIG_USOCKLND ]) # -# LP_CONDITIONALS +# LN_CONDITIONALS # -# AM_CONDITOINAL defines for portals +# AM_CONDITOINAL defines for lnet # -AC_DEFUN([LP_CONDITIONALS], -[AM_CONDITIONAL(BUILD_QSWNAL, test x$QSWNAL = "xqswnal") -AM_CONDITIONAL(BUILD_GMNAL, test x$GMNAL = "xgmnal") -AM_CONDITIONAL(BUILD_OPENIBNAL, test x$OPENIBNAL = "xopenibnal") -AM_CONDITIONAL(BUILD_IIBNAL, test x$IIBNAL = "xiibnal") -AM_CONDITIONAL(BUILD_RANAL, test x$RANAL = "xranal") +AC_DEFUN([LN_CONDITIONALS], +[AM_CONDITIONAL(BUILD_QSWLND, test x$QSWLND = "xqswlnd") +AM_CONDITIONAL(BUILD_GMLND, test x$GMLND = "xgmlnd") +AM_CONDITIONAL(BUILD_MXLND, test x$MXLND = "xmxlnd") +AM_CONDITIONAL(BUILD_O2IBLND, test x$O2IBLND = "xo2iblnd") +AM_CONDITIONAL(BUILD_OPENIBLND, test x$OPENIBLND = "xopeniblnd") +AM_CONDITIONAL(BUILD_CIBLND, test x$CIBLND = "xciblnd") +AM_CONDITIONAL(BUILD_IIBLND, test x$IIBLND = "xiiblnd") +AM_CONDITIONAL(BUILD_VIBLND, test x$VIBLND = "xviblnd") +AM_CONDITIONAL(BUILD_RALND, test x$RALND = "xralnd") +AM_CONDITIONAL(BUILD_PTLLND, test x$PTLLND = "xptllnd") +AM_CONDITIONAL(BUILD_UPTLLND, test x$UPTLLND = "xptllnd") +AM_CONDITIONAL(BUILD_USOCKLND, test x$USOCKLND = "xusocklnd") ]) # -# LP_CONFIG_FILES +# LN_CONFIG_FILES # # files that should be generated with AC_OUTPUT # -AC_DEFUN([LP_CONFIG_FILES], +AC_DEFUN([LN_CONFIG_FILES], [AC_CONFIG_FILES([ -portals/Kernelenv -portals/Makefile -portals/autoMakefile -portals/autoconf/Makefile -portals/doc/Makefile -portals/include/Makefile -portals/include/linux/Makefile -portals/include/portals/Makefile -portals/knals/Makefile -portals/knals/autoMakefile -portals/knals/gmnal/Makefile -portals/knals/gmnal/autoMakefile -portals/knals/iibnal/Makefile -portals/knals/iibnal/autoMakefile -portals/knals/lonal/Makefile -portals/knals/lonal/autoMakefile -portals/knals/openibnal/Makefile -portals/knals/openibnal/autoMakefile -portals/knals/qswnal/Makefile -portals/knals/qswnal/autoMakefile -portals/knals/ranal/Makefile -portals/knals/ranal/autoMakefile -portals/knals/socknal/Makefile -portals/knals/socknal/autoMakefile -portals/libcfs/Makefile -portals/libcfs/autoMakefile -portals/portals/Makefile -portals/portals/autoMakefile -portals/router/Makefile -portals/router/autoMakefile -portals/tests/Makefile -portals/tests/autoMakefile -portals/unals/Makefile -portals/utils/Makefile +lnet/Kernelenv +lnet/Makefile +lnet/autoMakefile +lnet/autoconf/Makefile +lnet/doc/Makefile +lnet/include/Makefile +lnet/include/lnet/Makefile +lnet/include/lnet/linux/Makefile +lnet/klnds/Makefile +lnet/klnds/autoMakefile +lnet/klnds/gmlnd/Makefile +lnet/klnds/mxlnd/autoMakefile +lnet/klnds/mxlnd/Makefile +lnet/klnds/gmlnd/autoMakefile +lnet/klnds/openiblnd/Makefile +lnet/klnds/openiblnd/autoMakefile +lnet/klnds/o2iblnd/Makefile +lnet/klnds/o2iblnd/autoMakefile +lnet/klnds/ciblnd/Makefile +lnet/klnds/ciblnd/autoMakefile +lnet/klnds/iiblnd/Makefile +lnet/klnds/iiblnd/autoMakefile +lnet/klnds/viblnd/Makefile +lnet/klnds/viblnd/autoMakefile +lnet/klnds/qswlnd/Makefile +lnet/klnds/qswlnd/autoMakefile +lnet/klnds/ralnd/Makefile +lnet/klnds/ralnd/autoMakefile +lnet/klnds/socklnd/Makefile +lnet/klnds/socklnd/autoMakefile +lnet/klnds/ptllnd/Makefile +lnet/klnds/ptllnd/autoMakefile +lnet/lnet/Makefile +lnet/lnet/autoMakefile +lnet/selftest/Makefile +lnet/selftest/autoMakefile +lnet/ulnds/Makefile +lnet/ulnds/autoMakefile +lnet/ulnds/socklnd/Makefile +lnet/ulnds/ptllnd/Makefile +lnet/utils/Makefile +]) +case $lb_target_os in + darwin) + AC_CONFIG_FILES([ +lnet/include/lnet/darwin/Makefile ]) + ;; +esac ])