From: eeb Date: Tue, 26 Apr 2005 02:52:47 +0000 (+0000) Subject: * temporarily disabled socknal explicit peer table setup until lconf X-Git-Tag: v1_7_100~1^25~6^2~277 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=49116c0833fc629d2daf725a3a39acfb650d8ae1;p=fs%2Flustre-release.git * temporarily disabled socknal explicit peer table setup until lconf fixed to stop doing it * added socknal support for peer IP == local_address(NID) * added CFS_MODULE_PARM macros * changed socknal and qswnal tunables to allow setting via module parameters --- diff --git a/lnet/include/libcfs/darwin/kp30.h b/lnet/include/libcfs/darwin/kp30.h index 7f765e1..a95d769 100644 --- a/lnet/include/libcfs/darwin/kp30.h +++ b/lnet/include/libcfs/darwin/kp30.h @@ -47,6 +47,14 @@ #define printk(format, args...) printf(format, ## args) +/******************************************************************************/ +/* Module parameter support */ +#define CFS_MODULE_PARM(name, t, type, perm, desc) \ + this should force a syntax error + +#define CFS_SYSFS_MODULE_PARM 0 /* no sysfs access to module parameters */ +/******************************************************************************/ + #else /* !__KERNEL__ */ # include # include diff --git a/lnet/include/libcfs/kp30.h b/lnet/include/libcfs/kp30.h index 0636aa1..9a1ece4 100644 --- a/lnet/include/libcfs/kp30.h +++ b/lnet/include/libcfs/kp30.h @@ -208,6 +208,8 @@ void portals_debug_dumplog(void); #define CLASSERT(cond) ({ switch(42) { case (cond): case 0: break; } }) /* support decl needed both by kernel and liblustre */ +int libcfs_isknown_nettype(int type); +char *libcfs_nettype2str(int type); char *libcfs_nid2str(ptl_nid_t nid); char *libcfs_id2str(ptl_process_id_t id); diff --git a/lnet/include/libcfs/linux/kp30.h b/lnet/include/libcfs/linux/kp30.h index e588c45..e0b3e60 100644 --- a/lnet/include/libcfs/linux/kp30.h +++ b/lnet/include/libcfs/linux/kp30.h @@ -135,6 +135,20 @@ do { \ #endif /******************************************************************************/ +/* Module parameter support */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)) +# define CFS_MODULE_PARM(name, t, type, perm, desc) \ + MODULE_PARM(name, t);\ + MODULE_PARM_DESC(name, desc) + +# define CFS_SYSFS_MODULE_PARM 0 /* no sysfs module parameters */ +#else +# define CFS_MODULE_PARM(name, t, type, perm, desc) \ + module_param(name, type, perm);\ + MODULE_PARM_DESC(name, desc) +# define CFS_SYSFS_MODULE_PARM 1 /* module parameters accessible via sysfs */ +#endif +/******************************************************************************/ #if (__GNUC__) /* Use the special GNU C __attribute__ hack to have the compiler check the diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index b50c1b9..f4e681a 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -407,7 +407,7 @@ extern void ptl_md_unlink(ptl_libmd_t *md); extern void ptl_md_deconstruct(ptl_libmd_t *lmd, ptl_md_t *umd); #ifdef __KERNEL__ -extern int ptl_register_nal(ptl_nal_t *nal); +extern void ptl_register_nal(ptl_nal_t *nal); extern void ptl_unregister_nal(ptl_nal_t *nal); #endif diff --git a/lnet/include/lnet/lib-p30.h b/lnet/include/lnet/lib-p30.h index b50c1b9..f4e681a 100644 --- a/lnet/include/lnet/lib-p30.h +++ b/lnet/include/lnet/lib-p30.h @@ -407,7 +407,7 @@ extern void ptl_md_unlink(ptl_libmd_t *md); extern void ptl_md_deconstruct(ptl_libmd_t *lmd, ptl_md_t *umd); #ifdef __KERNEL__ -extern int ptl_register_nal(ptl_nal_t *nal); +extern void ptl_register_nal(ptl_nal_t *nal); extern void ptl_unregister_nal(ptl_nal_t *nal); #endif diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 2ae92ed..0d33782 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -227,8 +227,7 @@ typedef struct ptl_nal int nal_refcount; /* # active instances */ /* fields initialised by the NAL */ - char *nal_name; /* NAL's type-name */ - int nal_type; + unsigned int nal_type; ptl_err_t (*nal_startup) (struct ptl_ni *ni, char **interfaces); void (*nal_shutdown) (struct ptl_ni *ni); diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c index 4e7ace5..0af6295 100644 --- a/lnet/klnds/gmlnd/gmlnd_api.c +++ b/lnet/klnds/gmlnd/gmlnd_api.c @@ -314,7 +314,7 @@ gmnal_startup(ptl_ni_t *ni, char **interfaces) nal_data->sysctl = NULL; nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0); - CDEBUG(D_INFO, "gmnal_init finished\n"); + CDEBUG(D_INFO, "finished\n"); global_nal_data = nal_data; @@ -322,7 +322,6 @@ gmnal_startup(ptl_ni_t *ni, char **interfaces) } ptl_nal_t the_gm_nal = { - .nal_name = "gm", .nal_type = GMNAL, .nal_startup = gmnal_startup, .nal_shutdown = gmnal_shutdown, @@ -337,13 +336,8 @@ ptl_nal_t the_gm_nal = { */ int gmnal_init(void) { - int rc; - - rc = ptl_register_nal(&the_gm_nal); - if (rc != PTL_OK) - CERROR("Can't register GMNAL: %d\n", rc); - - return (rc); + ptl_register_nal(&the_gm_nal); + return (0); } diff --git a/lnet/klnds/iiblnd/iiblnd.c b/lnet/klnds/iiblnd/iiblnd.c index fcbd1c1..dc92f36 100644 --- a/lnet/klnds/iiblnd/iiblnd.c +++ b/lnet/klnds/iiblnd/iiblnd.c @@ -24,7 +24,6 @@ #include "iibnal.h" ptl_nal_t kibnal_nal = { - .nal_name = "iib", .nal_type = IIBNAL, .nal_startup = kibnal_startup, .nal_shutdown = kibnal_shutdown, @@ -124,7 +123,7 @@ kibnal_check_advert (void) qry->OutputType = OutputTypeServiceRecord; qry->InputValue.ServiceRecordValue.ComponentMask = KIBNAL_SERVICE_KEY_MASK; svc = &qry->InputValue.ServiceRecordValue.ServiceRecord; - kibnal_set_service_keys(svc, kibnal_data.kib_nid); + kibnal_set_service_keys(svc, kibnal_data.kib_ni->ni_nid); frc = iibt_sd_query_port_fabric_information(kibnal_data.kib_sd, kibnal_data.kib_port_guid, @@ -159,7 +158,7 @@ static void fill_fod(FABRIC_OPERATION_DATA *fod, FABRIC_OPERATION_TYPE type) svc->RID.ServiceP_Key = kibnal_data.kib_port_pkey; svc->ServiceLease = 0xffffffff; - kibnal_set_service_keys(svc, kibnal_data.kib_nid); + kibnal_set_service_keys(svc, kibnal_data.kib_ni->ni_nid); } static int @@ -170,7 +169,7 @@ kibnal_advertise (void) FSTATUS frc; FSTATUS frc2; - LASSERT (kibnal_data.kib_nid != PTL_NID_ANY); + LASSERT (kibnal_data.kib_ni->ni_nid != PTL_NID_ANY); PORTAL_ALLOC(fod, sizeof(*fod)); if (fod == NULL) @@ -190,7 +189,7 @@ kibnal_advertise (void) if (frc != FSUCCESS && frc != FPENDING) { CERROR ("Immediate error %d advertising NID "LPX64"\n", - frc, kibnal_data.kib_nid); + frc, kibnal_data.kib_ni->ni_nid); goto out; } @@ -199,7 +198,7 @@ kibnal_advertise (void) frc = frc2; if (frc != FSUCCESS) CERROR ("Error %d advertising BUD "LPX64"\n", - frc, kibnal_data.kib_nid); + frc, kibnal_data.kib_ni->ni_nid); out: PORTAL_FREE(fod, sizeof(*fod)); return (frc == FSUCCESS) ? 0 : -EINVAL; @@ -213,7 +212,7 @@ kibnal_unadvertise (int expect_success) FSTATUS frc; FSTATUS frc2; - LASSERT (kibnal_data.kib_nid != PTL_NID_ANY); + LASSERT (kibnal_data.kib_ni->ni_nid != PTL_NID_ANY); PORTAL_ALLOC(fod, sizeof(*fod)); if (fod == NULL) @@ -232,7 +231,7 @@ kibnal_unadvertise (int expect_success) if (frc != FSUCCESS && frc != FPENDING) { CERROR ("Immediate error %d unadvertising NID "LPX64"\n", - frc, kibnal_data.kib_nid); + frc, kibnal_data.kib_ni->ni_nid); goto out; } @@ -243,10 +242,10 @@ kibnal_unadvertise (int expect_success) if (expect_success) CERROR("Error %d unadvertising NID "LPX64"\n", - frc2, kibnal_data.kib_nid); + frc2, kibnal_data.kib_ni->ni_nid); else CWARN("Removed conflicting NID "LPX64"\n", - kibnal_data.kib_nid); + kibnal_data.kib_ni->ni_nid); out: PORTAL_FREE(fod, sizeof(*fod)); } @@ -255,27 +254,26 @@ static int kibnal_set_mynid(ptl_nid_t nid) { struct timeval tv; - ptl_ni_t *ni = kibnal_data.kib_ni; int rc; FSTATUS frc; CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n", - nid, ni->ni_nid); + nid, kibnal_data.kib_ni->ni_nid); do_gettimeofday(&tv); down (&kibnal_data.kib_nid_mutex); - if (nid == kibnal_data.kib_nid) { + if (nid == kibnal_data.kib_ni->ni_nid) { /* no change of NID */ up (&kibnal_data.kib_nid_mutex); return (0); } CDEBUG(D_NET, "NID "LPX64"("LPX64")\n", - kibnal_data.kib_nid, nid); + kibnal_data.kib_ni->ni_nid, nid); - if (kibnal_data.kib_nid != PTL_NID_ANY) { + if (kibnal_data.kib_ni->ni_nid != PTL_NID_ANY) { kibnal_unadvertise (1); @@ -290,7 +288,7 @@ kibnal_set_mynid(ptl_nid_t nid) kibnal_data.kib_cep = NULL; } - kibnal_data.kib_nid = ni->ni_nid = nid; + kibnal_data.kib_ni->ni_nid = nid; kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec; /* Delete all existing peers and their connections after new @@ -298,7 +296,7 @@ kibnal_set_mynid(ptl_nid_t nid) * new world. */ kibnal_del_peer (PTL_NID_ANY, 0); - if (kibnal_data.kib_nid == PTL_NID_ANY) { + if (kibnal_data.kib_ni->ni_nid == PTL_NID_ANY) { /* No new NID to install */ up (&kibnal_data.kib_nid_mutex); return (0); @@ -343,7 +341,7 @@ kibnal_set_mynid(ptl_nid_t nid) kibnal_del_peer (PTL_NID_ANY, 0); } - kibnal_data.kib_nid = PTL_NID_ANY; + kibnal_data.kib_ni->ni_nid = PTL_NID_ANY; up (&kibnal_data.kib_nid_mutex); return (rc); } @@ -1350,7 +1348,7 @@ kibnal_startup (ptl_ni_t *ni, char **interfaces) init_MUTEX (&kibnal_data.kib_nid_mutex); init_MUTEX_LOCKED (&kibnal_data.kib_nid_signal); - kibnal_data.kib_nid = PTL_NID_ANY; + kibnal_data.kib_ni->ni_nid = PTL_NID_ANY; rwlock_init(&kibnal_data.kib_global_lock); @@ -1672,11 +1670,7 @@ kibnal_module_init (void) /* Initialise dynamic tunables to defaults once only */ kibnal_tunables.kib_io_timeout = IBNAL_IO_TIMEOUT; - rc = ptl_register_nal(&kibnal_nal); - if (rc != PTL_OK) { - CERROR("Can't register IBNAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } + ptl_register_nal(&kibnal_nal); #ifdef CONFIG_SYSCTL /* Press on regardless even if registering sysctl doesn't work */ diff --git a/lnet/klnds/iiblnd/iiblnd.h b/lnet/klnds/iiblnd/iiblnd.h index 05b36fc..1354ba2 100644 --- a/lnet/klnds/iiblnd/iiblnd.h +++ b/lnet/klnds/iiblnd/iiblnd.h @@ -174,7 +174,6 @@ typedef struct __u64 kib_service_id; /* service number I listen on */ __u64 kib_port_guid; /* my GUID (lo 64 of GID)*/ __u16 kib_port_pkey; /* my pkey, whatever that is */ - ptl_nid_t kib_nid; /* my NID */ struct semaphore kib_nid_mutex; /* serialise NID ops */ struct semaphore kib_nid_signal; /* signal completion */ IB_HANDLE kib_cep; /* connection end point */ diff --git a/lnet/klnds/iiblnd/iiblnd_cb.c b/lnet/klnds/iiblnd/iiblnd_cb.c index 2026e4c..45655a9 100644 --- a/lnet/klnds/iiblnd/iiblnd_cb.c +++ b/lnet/klnds/iiblnd/iiblnd_cb.c @@ -2413,7 +2413,7 @@ kibnal_listen_callback(IB_HANDLE cep, CM_CONN_INFO *info, void *arg) .wcr_magic = cpu_to_le32(IBNAL_MSG_MAGIC), .wcr_version = cpu_to_le16(IBNAL_MSG_VERSION), .wcr_queue_depth = cpu_to_le32(IBNAL_MSG_QUEUE_SIZE), - .wcr_nid = cpu_to_le64(kibnal_data.kib_nid), + .wcr_nid = cpu_to_le64(kibnal_data.kib_ni->ni_nid), .wcr_incarnation = cpu_to_le64(kibnal_data.kib_incarnation), }; @@ -2513,7 +2513,7 @@ kibnal_pathreq_callback (void *arg, QUERY *query, .wcr_magic = cpu_to_le32(IBNAL_MSG_MAGIC), .wcr_version = cpu_to_le16(IBNAL_MSG_VERSION), .wcr_queue_depth = cpu_to_le16(IBNAL_MSG_QUEUE_SIZE), - .wcr_nid = cpu_to_le64(kibnal_data.kib_nid), + .wcr_nid = cpu_to_le64(kibnal_data.kib_ni->ni_nid), .wcr_incarnation = cpu_to_le64(kibnal_data.kib_incarnation), }; diff --git a/lnet/klnds/lolnd/lolnd.c b/lnet/klnds/lolnd/lolnd.c index 815d9f5..1f63d25 100644 --- a/lnet/klnds/lolnd/lolnd.c +++ b/lnet/klnds/lolnd/lolnd.c @@ -22,7 +22,6 @@ #include "lonal.h" ptl_nal_t klonal_nal = { - .nal_name = "lo", .nal_type = LONAL, .nal_startup = klonal_startup, .nal_shutdown = klonal_shutdown, @@ -75,14 +74,7 @@ klonal_finalise (void) static int __init klonal_initialise (void) { - int rc; - - rc = ptl_register_nal(&klonal_nal); - if (rc != PTL_OK) { - CERROR("Can't register LONAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } - + ptl_register_nal(&klonal_nal); return (0); } diff --git a/lnet/klnds/openiblnd/openiblnd.c b/lnet/klnds/openiblnd/openiblnd.c index 7076f4d..582b3de 100644 --- a/lnet/klnds/openiblnd/openiblnd.c +++ b/lnet/klnds/openiblnd/openiblnd.c @@ -24,7 +24,6 @@ #include "openibnal.h" ptl_nal_t kibnal_nal = { - .nal_name = "openib", .nal_type = OPENIBNAL, .nal_startup = kibnal_startup, .nal_shutdown = kibnal_shutdown, @@ -931,22 +930,21 @@ kibnal_stop_ib_listener (void) int kibnal_set_mynid (ptl_nid_t nid) { - ptl_ni_t *ni = kibnal_data.kib_ni; int rc; CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n", - nid, ni->ni_nid); + nid, kibnal_data.kib_ni->ni_nid); down (&kibnal_data.kib_nid_mutex); - if (nid == kibnal_data.kib_nid) { + if (nid == kibnal_data.kib_ni->ni_nid) { /* no change of NID */ up (&kibnal_data.kib_nid_mutex); return (0); } CDEBUG(D_NET, "NID "LPX64"("LPX64")\n", - kibnal_data.kib_nid, nid); + kibnal_data.kib_ni->ni_nid, nid); if (kibnal_data.kib_listener_sock != NULL) kibnal_stop_ip_listener(1); @@ -954,7 +952,7 @@ kibnal_set_mynid (ptl_nid_t nid) if (kibnal_data.kib_listen_handle != NULL) kibnal_stop_ib_listener(); - ni->ni_nid = nid; + kibnal_data.kib_ni->ni_nid = nid; kibnal_data.kib_incarnation++; mb(); /* Delete all existing peers and their connections after new @@ -962,7 +960,7 @@ kibnal_set_mynid (ptl_nid_t nid) * world. */ kibnal_del_peer (PTL_NID_ANY, 0); - if (ni->ni_nid != PTL_NID_ANY) { + if (kibnal_data.kib_ni->ni_nid != PTL_NID_ANY) { /* got a new NID to install */ rc = kibnal_start_ib_listener(); if (rc != 0) { @@ -983,7 +981,7 @@ kibnal_set_mynid (ptl_nid_t nid) failed_1: kibnal_stop_ib_listener(); failed_0: - ni->ni_nid = PTL_NID_ANY; + kibnal_data.kib_ni->ni_nid = PTL_NID_ANY; kibnal_data.kib_incarnation++; mb(); kibnal_del_peer (PTL_NID_ANY, 0); @@ -2149,11 +2147,7 @@ kibnal_module_init (void) kibnal_tunables.kib_backlog = IBNAL_BACKLOG; kibnal_tunables.kib_port = IBNAL_PORT; - rc = ptl_register_nal(&kibnal_nal); - if (rc != PTL_OK) { - CERROR("Can't register IBNAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } + ptl_register_nal(&kibnal_nal); kibnal_tunables.kib_sysctl = register_sysctl_table (kibnal_top_ctl_table, 0); diff --git a/lnet/klnds/openiblnd/openiblnd.h b/lnet/klnds/openiblnd/openiblnd.h index 1b02744..9641fbb 100644 --- a/lnet/klnds/openiblnd/openiblnd.h +++ b/lnet/klnds/openiblnd/openiblnd.h @@ -152,7 +152,6 @@ typedef struct tTS_IB_GID kib_svc_gid; /* device/port GID */ __u16 kib_svc_pkey; /* device/port pkey */ - ptl_nid_t kib_nid; /* my NID */ struct semaphore kib_nid_mutex; /* serialise NID ops */ struct semaphore kib_listener_signal; /* signal IP listener completion */ struct socket *kib_listener_sock; /* IP listener's socket */ diff --git a/lnet/klnds/qswlnd/Makefile.in b/lnet/klnds/qswlnd/Makefile.in index d27240c..6cb6c30 100644 --- a/lnet/klnds/qswlnd/Makefile.in +++ b/lnet/klnds/qswlnd/Makefile.in @@ -1,5 +1,5 @@ MODULES := kqswnal -kqswnal-objs := qswnal.o qswnal_cb.o +kqswnal-objs := qswnal.o qswnal_cb.o qswnal_modparams.o EXTRA_POST_CFLAGS := @QSWCPPFLAGS@ -I/usr/include diff --git a/lnet/klnds/qswlnd/qswlnd.c b/lnet/klnds/qswlnd/qswlnd.c index 9560950..8ccf8f8 100644 --- a/lnet/klnds/qswlnd/qswlnd.c +++ b/lnet/klnds/qswlnd/qswlnd.c @@ -24,7 +24,6 @@ ptl_nal_t kqswnal_nal = { - .nal_name = "elan", .nal_type = QSWNAL, .nal_startup = kqswnal_startup, .nal_shutdown = kqswnal_shutdown, @@ -35,7 +34,6 @@ ptl_nal_t kqswnal_nal = }; kqswnal_data_t kqswnal_data; -kqswnal_tunables_t kqswnal_tunables; kpr_nal_interface_t kqswnal_router_interface = { kprni_nalid: QSWNAL, @@ -44,28 +42,6 @@ kpr_nal_interface_t kqswnal_router_interface = { kprni_notify: NULL, /* we're connectionless */ }; -#if CONFIG_SYSCTL -#define QSWNAL_SYSCTL 201 - -#define QSWNAL_SYSCTL_OPTIMIZED_GETS 1 -#define QSWNAL_SYSCTL_OPTIMIZED_PUTS 2 - -static ctl_table kqswnal_ctl_table[] = { - {QSWNAL_SYSCTL_OPTIMIZED_PUTS, "optimized_puts", - &kqswnal_tunables.kqn_optimized_puts, sizeof (int), - 0644, NULL, &proc_dointvec}, - {QSWNAL_SYSCTL_OPTIMIZED_GETS, "optimized_gets", - &kqswnal_tunables.kqn_optimized_gets, sizeof (int), - 0644, NULL, &proc_dointvec}, - {0} -}; - -static ctl_table kqswnal_top_ctl_table[] = { - {QSWNAL_SYSCTL, "qswnal", NULL, 0, 0555, kqswnal_ctl_table}, - {0} -}; -#endif - int kqswnal_get_tx_desc (struct portals_cfg *pcfg) { @@ -297,8 +273,10 @@ kqswnal_shutdown(ptl_ni_t *ni) { elan3_dvma_unload(kqswnal_data.kqn_ep->DmaState, kqswnal_data.kqn_eprxdmahandle, 0, - KQSW_NRXMSGPAGES_SMALL * KQSW_NRXMSGS_SMALL + - KQSW_NRXMSGPAGES_LARGE * KQSW_NRXMSGS_LARGE); + KQSW_NRXMSGPAGES_SMALL * + (*kqswnal_tunables.kqn_nrxmsgs_small) + + KQSW_NRXMSGPAGES_LARGE * + (*kqswnal_tunables.kqn_nrxmsgs_large)); elan3_dma_release(kqswnal_data.kqn_ep->DmaState, kqswnal_data.kqn_eprxdmahandle); @@ -308,8 +286,8 @@ kqswnal_shutdown(ptl_ni_t *ni) { elan3_dvma_unload(kqswnal_data.kqn_ep->DmaState, kqswnal_data.kqn_eptxdmahandle, 0, - KQSW_NTXMSGPAGES * (KQSW_NTXMSGS + - KQSW_NNBLK_TXMSGS)); + KQSW_NTXMSGPAGES * (*kqswnal_tunables.kqn_ntxmsgs + + *kqswnal_tunables.kqn_nnblk_txmsgs)); elan3_dma_release(kqswnal_data.kqn_ep->DmaState, kqswnal_data.kqn_eptxdmahandle); @@ -444,9 +422,10 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) /**********************************************************************/ /* Get the receivers */ - kqswnal_data.kqn_eprx_small = ep_alloc_rcvr (kqswnal_data.kqn_ep, - EP_MSG_SVC_PORTALS_SMALL, - KQSW_EP_ENVELOPES_SMALL); + kqswnal_data.kqn_eprx_small = + ep_alloc_rcvr (kqswnal_data.kqn_ep, + EP_MSG_SVC_PORTALS_SMALL, + *kqswnal_tunables.kqn_ep_envelopes_small); if (kqswnal_data.kqn_eprx_small == NULL) { CERROR ("Can't install small msg receiver\n"); @@ -454,9 +433,10 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) return (PTL_NO_SPACE); } - kqswnal_data.kqn_eprx_large = ep_alloc_rcvr (kqswnal_data.kqn_ep, - EP_MSG_SVC_PORTALS_LARGE, - KQSW_EP_ENVELOPES_LARGE); + kqswnal_data.kqn_eprx_large = + ep_alloc_rcvr (kqswnal_data.kqn_ep, + EP_MSG_SVC_PORTALS_LARGE, + *kqswnal_tunables.kqn_ep_envelopes_large); if (kqswnal_data.kqn_eprx_large == NULL) { CERROR ("Can't install large msg receiver\n"); @@ -471,7 +451,8 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) #if MULTIRAIL_EKC kqswnal_data.kqn_ep_tx_nmh = ep_dvma_reserve(kqswnal_data.kqn_ep, - KQSW_NTXMSGPAGES*(KQSW_NTXMSGS+KQSW_NNBLK_TXMSGS), + KQSW_NTXMSGPAGES*(*kqswnal_tunables.kqn_ntxmsgs+ + *kqswnal_tunables.kqn_nnblk_txmsgs), EP_PERM_WRITE); if (kqswnal_data.kqn_ep_tx_nmh == NULL) { CERROR("Can't reserve tx dma space\n"); @@ -485,7 +466,8 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) dmareq.Perm = ELAN_PERM_REMOTEWRITE; rc = elan3_dma_reserve(kqswnal_data.kqn_ep->DmaState, - KQSW_NTXMSGPAGES*(KQSW_NTXMSGS+KQSW_NNBLK_TXMSGS), + KQSW_NTXMSGPAGES*(*kqswnal_tunables.kqn_ntxmsgs+ + *kqswnal_tunables.kqn_nnblk_txmsgs), &dmareq, &kqswnal_data.kqn_eptxdmahandle); if (rc != DDI_SUCCESS) { @@ -499,8 +481,10 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) #if MULTIRAIL_EKC kqswnal_data.kqn_ep_rx_nmh = ep_dvma_reserve(kqswnal_data.kqn_ep, - KQSW_NRXMSGPAGES_SMALL * KQSW_NRXMSGS_SMALL + - KQSW_NRXMSGPAGES_LARGE * KQSW_NRXMSGS_LARGE, + KQSW_NRXMSGPAGES_SMALL * + (*kqswnal_tunables.kqn_nrxmsgs_small) + + KQSW_NRXMSGPAGES_LARGE * + (*kqswnal_tunables.kqn_nrxmsgs_large), EP_PERM_WRITE); if (kqswnal_data.kqn_ep_tx_nmh == NULL) { CERROR("Can't reserve rx dma space\n"); @@ -514,8 +498,10 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) dmareq.Perm = ELAN_PERM_REMOTEWRITE; rc = elan3_dma_reserve (kqswnal_data.kqn_ep->DmaState, - KQSW_NRXMSGPAGES_SMALL * KQSW_NRXMSGS_SMALL + - KQSW_NRXMSGPAGES_LARGE * KQSW_NRXMSGS_LARGE, + KQSW_NRXMSGPAGES_SMALL * + (*kqswnal_tunables.kqn_nrxmsgs_small) + + KQSW_NRXMSGPAGES_LARGE * + (*kqswnal_tunables.kqn_nrxmsgs_large), &dmareq, &kqswnal_data.kqn_eprxdmahandle); if (rc != DDI_SUCCESS) { @@ -528,7 +514,7 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) /* Allocate/Initialise transmit descriptors */ kqswnal_data.kqn_txds = NULL; - for (i = 0; i < (KQSW_NTXMSGS + KQSW_NNBLK_TXMSGS); i++) + for (i = 0; i < (*kqswnal_tunables.kqn_ntxmsgs + *kqswnal_tunables.kqn_nnblk_txmsgs); i++) { int premapped_pages; int basepage = i * KQSW_NTXMSGPAGES; @@ -573,7 +559,7 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) #if MULTIRAIL_EKC ktx->ktx_rail = -1; /* unset rail */ #endif - ktx->ktx_isnblk = (i >= KQSW_NTXMSGS); + ktx->ktx_isnblk = (i >= *kqswnal_tunables.kqn_ntxmsgs); list_add_tail (&ktx->ktx_list, ktx->ktx_isnblk ? &kqswnal_data.kqn_nblk_idletxds : &kqswnal_data.kqn_idletxds); @@ -583,7 +569,7 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) /* Allocate/Initialise receive descriptors */ kqswnal_data.kqn_rxds = NULL; elan_page_idx = 0; - for (i = 0; i < KQSW_NRXMSGS_SMALL + KQSW_NRXMSGS_LARGE; i++) + for (i = 0; i < *kqswnal_tunables.kqn_nrxmsgs_small + *kqswnal_tunables.kqn_nrxmsgs_large; i++) { #if MULTIRAIL_EKC EP_NMD elanbuffer; @@ -602,7 +588,7 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) krx->krx_alloclist = kqswnal_data.kqn_rxds; kqswnal_data.kqn_rxds = krx; - if (i < KQSW_NRXMSGS_SMALL) + if (i < *kqswnal_tunables.kqn_nrxmsgs_small) { krx->krx_npages = KQSW_NRXMSGPAGES_SMALL; krx->krx_eprx = kqswnal_data.kqn_eprx_small; @@ -658,8 +644,8 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) } } LASSERT (elan_page_idx == - (KQSW_NRXMSGS_SMALL * KQSW_NRXMSGPAGES_SMALL) + - (KQSW_NRXMSGS_LARGE * KQSW_NRXMSGPAGES_LARGE)); + (*kqswnal_tunables.kqn_nrxmsgs_small * KQSW_NRXMSGPAGES_SMALL) + + (*kqswnal_tunables.kqn_nrxmsgs_large * KQSW_NRXMSGPAGES_LARGE)); /**********************************************************************/ /* Queue receives, now that it's OK to run their completion callbacks */ @@ -721,33 +707,19 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) void __exit kqswnal_finalise (void) { -#if CONFIG_SYSCTL - if (kqswnal_tunables.kqn_sysctl != NULL) - unregister_sysctl_table (kqswnal_tunables.kqn_sysctl); -#endif ptl_unregister_nal(&kqswnal_nal); + kqswnal_tunables_fini(); } static int __init kqswnal_initialise (void) { - int rc; - - /* Initialise dynamic tunables to defaults once only */ - kqswnal_tunables.kqn_optimized_puts = KQSW_OPTIMIZED_PUTS; - kqswnal_tunables.kqn_optimized_gets = KQSW_OPTIMIZED_GETS; + int rc = kqswnal_tunables_init(); - rc = ptl_register_nal(&kqswnal_nal); - if (rc != PTL_OK) { - CERROR("Can't register QSWNAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } + if (rc != 0) + return rc; -#if CONFIG_SYSCTL - /* Press on regardless even if registering sysctl doesn't work */ - kqswnal_tunables.kqn_sysctl = - register_sysctl_table (kqswnal_top_ctl_table, 0); -#endif + ptl_register_nal(&kqswnal_nal); return (0); } diff --git a/lnet/klnds/qswlnd/qswlnd.h b/lnet/klnds/qswlnd/qswlnd.h index ca226ea..005407b 100644 --- a/lnet/klnds/qswlnd/qswlnd.h +++ b/lnet/klnds/qswlnd/qswlnd.h @@ -110,13 +110,12 @@ typedef unsigned long kqsw_csum_t; #define KQSW_OPTIMIZED_GETS 1 /* optimize gets >= this size */ #define KQSW_OPTIMIZED_PUTS (32<<10) /* optimize puts >= this size */ -#define KQSW_COPY_SMALL_FWD 0 /* copy small fwd messages to pre-mapped buffer? */ /* * derived constants */ -#define KQSW_TX_BUFFER_SIZE (KQSW_HDR_SIZE + KQSW_TX_MAXCONTIG) +#define KQSW_TX_BUFFER_SIZE (KQSW_HDR_SIZE + *kqswnal_tunables.kqn_tx_maxcontig) /* The pre-allocated tx buffer (hdr + small payload) */ #define KQSW_NTXMSGPAGES (btopr(KQSW_TX_BUFFER_SIZE) + 1 + btopr(KQSW_MAXPAYLOAD) + 1) @@ -209,11 +208,18 @@ typedef struct kqswnal_tx typedef struct { - /* dynamic tunables... */ - int kqn_optimized_puts; /* optimized PUTs? */ - int kqn_optimized_gets; /* optimized GETs? */ -#if CONFIG_SYSCTL - struct ctl_table_header *kqn_sysctl; /* sysctl interface */ + int *kqn_tx_maxcontig; /* maximum payload to defrag */ + int *kqn_ntxmsgs; /* # normal tx msgs */ + int *kqn_nnblk_txmsgs; /* # reserved tx msgs */ + int *kqn_nrxmsgs_large; /* # 'large' rx msgs */ + int *kqn_ep_envelopes_large; /* # 'large' rx ep envelopes */ + int *kqn_nrxmsgs_small; /* # 'small' rx msgs */ + int *kqn_ep_envelopes_small; /* # 'small' rx ep envelopes */ + int *kqn_optimized_puts; /* optimized PUTs? */ + int *kqn_optimized_gets; /* optimized GETs? */ + +#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM + struct ctl_table_header *kqn_sysctl; /* sysctl interface */ #endif } kqswnal_tunables_t; @@ -393,4 +399,7 @@ ptl_err_t kqswnal_recv_pages(ptl_ni_t *ni, void *private, ptl_kiov_t *kiov, size_t offset, size_t mlen, size_t rlen); +int kqswnal_tunables_init(void); +void kqswnal_tunables_fini(void); + #endif /* _QSWNAL_H */ diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c index a000c5f..7f2b1c9 100644 --- a/lnet/klnds/qswlnd/qswlnd_cb.c +++ b/lnet/klnds/qswlnd/qswlnd_cb.c @@ -1132,10 +1132,11 @@ kqswnal_sendmsg (ptl_ni_t *ni, if (nid == targetnid && /* not forwarding */ ((type == PTL_MSG_GET && /* optimize GET? */ kqswnal_tunables.kqn_optimized_gets != 0 && - le32_to_cpu(hdr->msg.get.sink_length) >= kqswnal_tunables.kqn_optimized_gets) || + le32_to_cpu(hdr->msg.get.sink_length) >= + *kqswnal_tunables.kqn_optimized_gets) || (type == PTL_MSG_PUT && /* optimize PUT? */ kqswnal_tunables.kqn_optimized_puts != 0 && - payload_nob >= kqswnal_tunables.kqn_optimized_puts))) { + payload_nob >= *kqswnal_tunables.kqn_optimized_puts))) { ptl_libmd_t *md = ptlmsg->msg_md; kqswnal_remotemd_t *rmd = (kqswnal_remotemd_t *)(ktx->ktx_buffer + KQSW_HDR_SIZE); @@ -1190,7 +1191,7 @@ kqswnal_sendmsg (ptl_ni_t *ni, * responsibility now, whatever happens. */ } - } else if (payload_nob <= KQSW_TX_MAXCONTIG) { + } else if (payload_nob <= *kqswnal_tunables.kqn_tx_maxcontig) { /* small message: single frag copied into the pre-mapped buffer */ @@ -1341,7 +1342,7 @@ kqswnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd) ktx->ktx_args[0] = fwd; ktx->ktx_nfrag = ktx->ktx_firsttmpfrag = 1; - if (nob <= KQSW_TX_MAXCONTIG) + if (nob <= *kqswnal_tunables.kqn_tx_maxcontig) { /* send payload from ktx's pre-mapped contiguous buffer */ #if MULTIRAIL_EKC diff --git a/lnet/klnds/qswlnd/qswlnd_modparams.c b/lnet/klnds/qswlnd/qswlnd_modparams.c new file mode 100644 index 0000000..7c47097 --- /dev/null +++ b/lnet/klnds/qswlnd/qswlnd_modparams.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2002-2004 Cluster File Systems, Inc. + * Author: Eric Barton + * + * This file is part of Portals, http://www.lustre.org + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#include "qswnal.h" + +static int tx_maxcontig = KQSW_TX_MAXCONTIG; +CFS_MODULE_PARM(tx_maxcontig, "i", int, 0444, + "maximum payload to de-fragment"); + +static int ntxmsgs = KQSW_NTXMSGS; +CFS_MODULE_PARM(ntxmsgs, "i", int, 0444, + "# 'normal' tx msg buffers"); + +static int nnblk_txmsgs = KQSW_NNBLK_TXMSGS; +CFS_MODULE_PARM(nnblk_txmsgs, "i", int, 0444, + "# 'reserved' tx msg buffers"); + +static int nrxmsgs_large = KQSW_NRXMSGS_LARGE; +CFS_MODULE_PARM(nrxmsgs_large, "i", int, 0444, + "# 'large' rx msg buffers"); + +static int ep_envelopes_large = KQSW_EP_ENVELOPES_LARGE; +CFS_MODULE_PARM(ep_envelopes_large, "i", int, 0444, + "# 'large' rx msg envelope buffers"); + +static int nrxmsgs_small = KQSW_NRXMSGS_SMALL; +CFS_MODULE_PARM(nrxmsgs_small, "i", int, 0444, + "# 'small' rx msg buffers"); + +static int ep_envelopes_small = KQSW_EP_ENVELOPES_SMALL; +CFS_MODULE_PARM(ep_envelopes_small, "i", int, 0444, + "# 'small' rx msg envelope buffers"); + +static int optimized_puts = KQSW_OPTIMIZED_PUTS; +CFS_MODULE_PARM(optimized_puts, "i", int, 0644, + "zero-copy puts >= this size"); + +static int optimized_gets = KQSW_OPTIMIZED_GETS; +CFS_MODULE_PARM(optimized_gets, "i", int, 0644, + "zero-copy gets >= this size"); + +kqswnal_tunables_t kqswnal_tunables = { + .kqn_tx_maxcontig = &tx_maxcontig, + .kqn_ntxmsgs = &ntxmsgs, + .kqn_nnblk_txmsgs = &nnblk_txmsgs, + .kqn_nrxmsgs_large = &nrxmsgs_large, + .kqn_ep_envelopes_large = &ep_envelopes_large, + .kqn_nrxmsgs_small = &nrxmsgs_small, + .kqn_ep_envelopes_small = &ep_envelopes_small, + .kqn_optimized_puts = &optimized_puts, + .kqn_optimized_gets = &optimized_gets, +}; + +#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM +static ctl_table kqswnal_ctl_table[] = { + {1, "tx_maxcontig", &tx_maxcontig, + sizeof (int), 0444, NULL, &proc_dointvec}, + {2, "ntxmsgs", &ntxmsgs, + sizeof (int), 0444, NULL, &proc_dointvec}, + {3, "nnblk_txmsgs", &nnblk_txmsgs, + sizeof (int), 0444, NULL, &proc_dointvec}, + {4, "nrxmsgs_large", &nrxmsgs_large, + sizeof (int), 0444, NULL, &proc_dointvec}, + {5, "ep_envelopes_large", &ep_envelopes_large, + sizeof (int), 0444, NULL, &proc_dointvec}, + {6, "nrxmsgs_small", &nrxmsgs_small, + sizeof (int), 0444, NULL, &proc_dointvec}, + {7, "ep_envelopes_small", &ep_envelopes_small, + sizeof (int), 0444, NULL, &proc_dointvec}, + {8, "optimized_puts", &optimized_puts, + sizeof (int), 0644, NULL, &proc_dointvec}, + {9, "optimized_gets", &optimized_gets, + sizeof (int), 0644, NULL, &proc_dointvec}, + {0} +}; + +static ctl_table kqswnal_top_ctl_table[] = { + {201, "qswnal", NULL, 0, 0555, kqswnal_ctl_table}, + {0} +}; + +int +kqswnal_tunables_init () +{ + kqswnal_tunables.kqn_sysctl = + register_sysctl_table(kqswnal_top_ctl_table, 0); + + if (kqswnal_tunables.kqn_sysctl == NULL) + CWARN("Can't setup /proc tunables\n"); + + return 0; +} + +void +kqswnal_tunables_fini () +{ + if (kqswnal_tunables.kqn_sysctl != NULL) + unregister_sysctl_table(kqswnal_tunables.kqn_sysctl); +} +#else +int +kqswnal_tunables_init () +{ + return 0; +} + +void +kqswnal_tunables_fini () +{ +} +#endif diff --git a/lnet/klnds/ralnd/ralnd.c b/lnet/klnds/ralnd/ralnd.c index 28aaa85..6caaca9 100644 --- a/lnet/klnds/ralnd/ralnd.c +++ b/lnet/klnds/ralnd/ralnd.c @@ -26,7 +26,6 @@ static int kranal_devids[] = {RAPK_MAIN_DEVICE_ID, RAPK_EXPANSION_DEVICE_ID}; ptl_nal_t kranal_nal = { - .nal_name = "ra", .nal_type = RANAL, .nal_startup = kranal_startup, .nal_shutdown = kranal_shutdown, @@ -2105,11 +2104,7 @@ kranal_module_init (void) kranal_tunables.kra_port = RANAL_PORT; kranal_tunables.kra_max_immediate = RANAL_MAX_IMMEDIATE; - rc = ptl_register_nal(&kranal_nal); - if (rc != PTL_OK) { - CERROR("Can't register RANAL: %d\n", rc); - return -ENOMEM; /* or something... */ - } + ptl_register_nal(&kranal_nal); kranal_tunables.kra_sysctl = register_sysctl_table(kranal_top_ctl_table, 0); diff --git a/lnet/klnds/socklnd/Makefile.in b/lnet/klnds/socklnd/Makefile.in index 7fe9638..f89ddb6 100644 --- a/lnet/klnds/socklnd/Makefile.in +++ b/lnet/klnds/socklnd/Makefile.in @@ -1,6 +1,6 @@ MODULES := ksocknal -ksocknal-objs := socknal.o socknal_cb.o socknal_lib-linux.o +ksocknal-objs := socknal.o socknal_cb.o socknal_modparams.o socknal_lib-linux.o # If you don't build with -O2, your modules won't insert, becahse htonl is # just special that way. diff --git a/lnet/klnds/socklnd/autoMakefile.am b/lnet/klnds/socklnd/autoMakefile.am index 5fc3fc7..28ed3d6 100644 --- a/lnet/klnds/socklnd/autoMakefile.am +++ b/lnet/klnds/socklnd/autoMakefile.am @@ -14,7 +14,7 @@ if DARWIN macos_PROGRAMS := ksocknal - nodist_ksocknal_SOURCES := socknal.c socknal_cb.c socknal_lib-darwin.c + nodist_ksocknal_SOURCES := socknal.c socknal_cb.c socknal_modparams.c socknal_lib-darwin.c DIST_SOURCES += socknal_lib-darwin.c socknal_lib-darwin.h ksocknal_CFLAGS := $(EXTRA_KCFLAGS) diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 540ff8d..b4a27bb 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -26,7 +26,6 @@ #include "socknal.h" ptl_nal_t ksocknal_nal = { - .nal_name = "tcp", .nal_type = SOCKNAL, .nal_startup = ksocknal_startup, .nal_shutdown = ksocknal_shutdown, @@ -36,7 +35,6 @@ ptl_nal_t ksocknal_nal = { .nal_recv_pages = ksocknal_recv_pages, }; ksock_nal_data_t ksocknal_data; -ksock_tunables_t ksocknal_tunables; kpr_nal_interface_t ksocknal_router_interface = { kprni_nalid: SOCKNAL, @@ -1118,7 +1116,7 @@ ksocknal_create_conn (ksock_route_t *route, struct socket *sock, int type) /* Set the deadline for the outgoing HELLO to drain */ conn->ksnc_tx_bufnob = SOCK_WMEM_QUEUED(sock); - conn->ksnc_tx_deadline = cfs_time_shift(ksocknal_tunables.ksnd_io_timeout); + conn->ksnc_tx_deadline = cfs_time_shift(*ksocknal_tunables.ksnd_timeout); mb(); /* order with adding to peer's conn list */ list_add (&conn->ksnc_list, &peer->ksnp_conns); @@ -1781,12 +1779,29 @@ ksocknal_cmd(struct portals_cfg *pcfg, void * private) break; } case NAL_CMD_ADD_PEER: { +#if 1 + CDEBUG(D_WARNING, "ADD_PEER: ignoring " + LPX64"@%u.%u.%u.%u:%d\n", + pcfg->pcfg_nid, + HIPQUAD(pcfg->pcfg_id), /* IP */ + pcfg->pcfg_misc); /* port */ + rc = 0; +#else rc = ksocknal_add_peer (pcfg->pcfg_nid, pcfg->pcfg_id, /* IP */ pcfg->pcfg_misc); /* port */ +#endif break; } case NAL_CMD_DEL_PEER: { + if (pcfg->pcfg_flags) { /* single_share */ + CDEBUG(D_WARNING, "DEL_PEER: ignoring " + LPX64"@%u.%u.%u.%u\n", + pcfg->pcfg_nid, + HIPQUAD(pcfg->pcfg_id)); /* IP */ + rc = 0; + break; + } rc = ksocknal_del_peer (pcfg->pcfg_nid, pcfg->pcfg_id, /* IP */ pcfg->pcfg_flags); /* single_share? */ @@ -2197,15 +2212,10 @@ ksocknal_startup (ptl_ni_t *ni, char **interfaces) void __exit ksocknal_module_fini (void) { -#ifdef CONFIG_SYSCTL - if (ksocknal_tunables.ksnd_sysctl != NULL) - unregister_sysctl_table (ksocknal_tunables.ksnd_sysctl); -#endif ptl_unregister_nal(&ksocknal_nal); + ksocknal_lib_tunables_fini(); } -extern cfs_sysctl_table_t ksocknal_top_ctl_table[]; - int __init ksocknal_module_init (void) { @@ -2213,54 +2223,15 @@ ksocknal_module_init (void) /* packet descriptor must fit in a router descriptor's scratchpad */ CLASSERT(sizeof (ksock_tx_t) <= sizeof (kprfd_scratch_t)); - /* the following must be sizeof(int) for proc_dointvec() */ - CLASSERT(sizeof (ksocknal_tunables.ksnd_io_timeout) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_eager_ack) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_typed_conns) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_min_bulk) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_buffer_size) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_nagle) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_keepalive_idle) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_keepalive_count) == sizeof (int)); - CLASSERT(sizeof (ksocknal_tunables.ksnd_keepalive_intvl) == sizeof (int)); -#if CPU_AFFINITY - CLASSERT(sizeof (ksocknal_tunables.ksnd_irq_affinity) == sizeof (int)); -#endif -#if SOCKNAL_ZC - CLASSERT(sizeof (ksocknal_tunables.ksnd_zc_min_frag) == sizeof (int)); -#endif /* check ksnr_connected/connecting field large enough */ CLASSERT(SOCKNAL_CONN_NTYPES <= 4); - /* Initialise dynamic tunables to defaults once only */ - ksocknal_tunables.ksnd_io_timeout = SOCKNAL_IO_TIMEOUT; - ksocknal_tunables.ksnd_eager_ack = SOCKNAL_EAGER_ACK; - ksocknal_tunables.ksnd_typed_conns = SOCKNAL_TYPED_CONNS; - ksocknal_tunables.ksnd_min_bulk = SOCKNAL_MIN_BULK; - ksocknal_tunables.ksnd_buffer_size = SOCKNAL_BUFFER_SIZE; - ksocknal_tunables.ksnd_nagle = SOCKNAL_NAGLE; - ksocknal_tunables.ksnd_keepalive_idle = SOCKNAL_KEEPALIVE_IDLE; - ksocknal_tunables.ksnd_keepalive_count = SOCKNAL_KEEPALIVE_COUNT; - ksocknal_tunables.ksnd_keepalive_intvl = SOCKNAL_KEEPALIVE_INTVL; -#if CPU_AFFINITY - ksocknal_tunables.ksnd_irq_affinity = SOCKNAL_IRQ_AFFINITY; -#endif -#if SOCKNAL_ZC - ksocknal_tunables.ksnd_zc_min_frag = SOCKNAL_ZC_MIN_FRAG; -#endif - - rc = ptl_register_nal(&ksocknal_nal); - if (rc != PTL_OK) { - CERROR("Can't register SOCKNAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } + rc = ksocknal_lib_tunables_init(); + if (rc != 0) + return rc; -#ifdef CONFIG_SYSCTL - /* Press on regardless even if registering sysctl doesn't work */ - ksocknal_tunables.ksnd_sysctl = - register_sysctl_table (ksocknal_top_ctl_table, 0); -#endif - return (0); + ptl_register_nal(&ksocknal_nal); + return 0; } MODULE_AUTHOR("Cluster File Systems, Inc. "); diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index e6fc03b..4c3006e 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -129,20 +129,24 @@ typedef struct typedef struct { - int ksnd_io_timeout; /* "stuck" socket timeout (seconds) */ - int ksnd_eager_ack; /* make TCP ack eagerly? */ - int ksnd_typed_conns; /* drive sockets by type? */ - int ksnd_min_bulk; /* smallest "large" message */ - int ksnd_buffer_size; /* socket buffer size */ - int ksnd_nagle; /* enable NAGLE? */ - int ksnd_irq_affinity; /* enable IRQ affinity? */ - int ksnd_keepalive_idle; /* # idle secs before 1st probe */ - int ksnd_keepalive_count; /* # probes */ - int ksnd_keepalive_intvl; /* time between probes */ + int *ksnd_timeout; /* "stuck" socket timeout (seconds) */ + int *ksnd_eager_ack; /* make TCP ack eagerly? */ + int *ksnd_typed_conns; /* drive sockets by type? */ + int *ksnd_min_bulk; /* smallest "large" message */ + int *ksnd_buffer_size; /* socket buffer size */ + int *ksnd_nagle; /* enable NAGLE? */ + int *ksnd_keepalive_idle; /* # idle secs before 1st probe */ + int *ksnd_keepalive_count; /* # probes */ + int *ksnd_keepalive_intvl; /* time between probes */ #if SOCKNAL_ZC - unsigned int ksnd_zc_min_frag; /* minimum zero copy frag size */ + unsigned int *ksnd_zc_min_frag; /* minimum zero copy frag size */ #endif +#if CPU_AFFINITY + int *ksnd_irq_affinity; /* enable IRQ affinity? */ +#endif +#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM cfs_sysctl_table_header_t *ksnd_sysctl; /* sysctl interface */ +#endif } ksock_tunables_t; typedef struct @@ -427,6 +431,7 @@ ptl_err_t ksocknal_recv_pages(ptl_ni_t *ni, void *private, extern void ksocknal_put_route (ksock_route_t *route); extern void ksocknal_put_peer (ksock_peer_t *peer); +extern int ksocknal_add_peer(ptl_nid_t nid, __u32 ip, int port); extern ksock_peer_t *ksocknal_find_peer_locked (ptl_nid_t nid); extern ksock_peer_t *ksocknal_get_peer (ptl_nid_t nid); extern int ksocknal_del_route (ptl_nid_t nid, __u32 ipaddr, @@ -478,3 +483,6 @@ extern int ksocknal_lib_get_conn_tunables (ksock_conn_t *conn, int *txmem, int *rxmem, int *nagle); extern int ksocknal_lib_connect_sock(struct socket **sockp, int *may_retry, ksock_route_t *route, int local_port); + +extern int ksocknal_lib_tunables_init(void); +extern void ksocknal_lib_tunables_fini(void); diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 643da78..c9bf19f 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -144,7 +144,8 @@ ksocknal_transmit (ksock_conn_t *conn, ksock_tx_t *tx) if (bufnob < conn->ksnc_tx_bufnob) { /* allocated send buffer bytes < computed; infer * something got ACKed */ - conn->ksnc_tx_deadline = cfs_time_shift(ksocknal_tunables.ksnd_io_timeout); + conn->ksnc_tx_deadline = + cfs_time_shift(*ksocknal_tunables.ksnd_timeout); conn->ksnc_peer->ksnp_last_alive = cfs_time_current(); conn->ksnc_tx_bufnob = bufnob; mb(); @@ -212,7 +213,8 @@ ksocknal_recv_iov (ksock_conn_t *conn) nob = rc; conn->ksnc_peer->ksnp_last_alive = cfs_time_current(); - conn->ksnc_rx_deadline = cfs_time_shift (ksocknal_tunables.ksnd_io_timeout); + conn->ksnc_rx_deadline = + cfs_time_shift(*ksocknal_tunables.ksnd_timeout); mb(); /* order with setting rx_started */ conn->ksnc_rx_started = 1; @@ -255,7 +257,8 @@ ksocknal_recv_kiov (ksock_conn_t *conn) nob = rc; conn->ksnc_peer->ksnp_last_alive = cfs_time_current(); - conn->ksnc_rx_deadline = cfs_time_shift (ksocknal_tunables.ksnd_io_timeout); + conn->ksnc_rx_deadline = + cfs_time_shift(*ksocknal_tunables.ksnd_timeout); mb(); /* order with setting rx_started */ conn->ksnc_rx_started = 1; @@ -320,7 +323,7 @@ ksocknal_receive (ksock_conn_t *conn) if (conn->ksnc_rx_nob_wanted == 0) { /* Completed a message segment (header or payload) */ - if ((ksocknal_tunables.ksnd_eager_ack & conn->ksnc_type) != 0 && + if ((*ksocknal_tunables.ksnd_eager_ack & conn->ksnc_type) != 0 && (conn->ksnc_rx_state == SOCKNAL_RX_BODY || conn->ksnc_rx_state == SOCKNAL_RX_BODY_FWD)) { /* Remind the socket to ack eagerly... */ @@ -510,6 +513,7 @@ ksocknal_launch_autoconnect_locked (ksock_route_t *route) spin_unlock_irqrestore (&ksocknal_data.ksnd_autoconnectd_lock, flags); } +#if 0 ksock_peer_t * ksocknal_find_target_peer_locked (ksock_tx_t *tx, ptl_nid_t nid) { @@ -543,6 +547,7 @@ ksocknal_find_target_peer_locked (ksock_tx_t *tx, ptl_nid_t nid) target_nid, libcfs_nid2str(target_nid)); return (NULL); } +#endif ksock_conn_t * ksocknal_find_conn_locked (ksock_tx_t *tx, ksock_peer_t *peer) @@ -569,7 +574,7 @@ ksocknal_find_conn_locked (ksock_tx_t *tx, ksock_peer_t *peer) fnob = nob; } - if (!ksocknal_tunables.ksnd_typed_conns) + if (!*ksocknal_tunables.ksnd_typed_conns) continue; switch (c->ksnc_type) { @@ -581,11 +586,11 @@ ksocknal_find_conn_locked (ksock_tx_t *tx, ksock_peer_t *peer) case SOCKNAL_CONN_BULK_IN: continue; case SOCKNAL_CONN_BULK_OUT: - if (tx->tx_nob < ksocknal_tunables.ksnd_min_bulk) + if (tx->tx_nob < *ksocknal_tunables.ksnd_min_bulk) continue; break; case SOCKNAL_CONN_CONTROL: - if (tx->tx_nob >= ksocknal_tunables.ksnd_min_bulk) + if (tx->tx_nob >= *ksocknal_tunables.ksnd_min_bulk) continue; break; } @@ -640,7 +645,8 @@ ksocknal_queue_tx_locked (ksock_tx_t *tx, ksock_conn_t *conn) if (list_empty(&conn->ksnc_tx_queue) && SOCK_WMEM_QUEUED(conn->ksnc_sock) == 0) { /* First packet starts the timeout */ - conn->ksnc_tx_deadline = cfs_time_shift(ksocknal_tunables.ksnd_io_timeout); + conn->ksnc_tx_deadline = + cfs_time_shift(*ksocknal_tunables.ksnd_timeout); conn->ksnc_tx_bufnob = 0; mb(); /* order with adding to tx_queue */ } @@ -671,7 +677,7 @@ ksocknal_find_connectable_route_locked (ksock_peer_t *peer) route = list_entry (tmp, ksock_route_t, ksnr_list); bits = route->ksnr_connected; - if (ksocknal_tunables.ksnd_typed_conns) { + if (*ksocknal_tunables.ksnd_typed_conns) { /* All typed connections established? */ if ((bits & KSNR_TYPED_ROUTES) == KSNR_TYPED_ROUTES) continue; @@ -719,6 +725,8 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid) ksock_conn_t *conn; ksock_route_t *route; rwlock_t *g_lock; + int create_peer = 0; + int rc; /* Ensure the frags we've been given EXACTLY match the number of * bytes we want to send. Many TCP/IP stacks disregard any total @@ -740,23 +748,31 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid) tx->tx_hdr = (ptl_hdr_t *)tx->tx_iov[0].iov_base; g_lock = &ksocknal_data.ksnd_global_lock; + + again: + if (create_peer) { + rc = ksocknal_add_peer(nid, (__u32)nid, 988); + if (rc != 0) { + CERROR("Can't add peer "LPX64": %d\n", + nid, rc); + return rc; + } + } + #if !SOCKNAL_ROUND_ROBIN read_lock (g_lock); - - peer = ksocknal_find_target_peer_locked (tx, nid); - if (peer == NULL) { - read_unlock (g_lock); - return (-EHOSTUNREACH); - } - - if (ksocknal_find_connectable_route_locked(peer) == NULL) { - conn = ksocknal_find_conn_locked (tx, peer); - if (conn != NULL) { - /* I've got no autoconnect routes that need to be - * connecting and I do have an actual connection... */ - ksocknal_queue_tx_locked (tx, conn); - read_unlock (g_lock); - return (0); +#warning "router issues" + peer = ksocknal_find_peer_locked(nid); + if (peer != NULL) { + if (ksocknal_find_connectable_route_locked(peer) == NULL) { + conn = ksocknal_find_conn_locked (tx, peer); + if (conn != NULL) { + /* I've got no autoconnect routes that need to be + * connecting and I do have an actual connection... */ + ksocknal_queue_tx_locked (tx, conn); + read_unlock (g_lock); + return (0); + } } } @@ -765,10 +781,18 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid) #endif write_lock_irqsave(g_lock, flags); - peer = ksocknal_find_target_peer_locked (tx, nid); +#warning "router issues" + peer = ksocknal_find_peer_locked(nid); if (peer == NULL) { write_unlock_irqrestore(g_lock, flags); - return (-EHOSTUNREACH); + + if (create_peer) { + CERROR("Can't find peer "LPX64"\n", nid); + return -ENOENT; + } + + create_peer = 1; + goto again; } for (;;) { @@ -798,6 +822,8 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid) } write_unlock_irqrestore (g_lock, flags); + + CERROR("Peer entry with no routes: "LPX64"\n", nid); return (-EHOSTUNREACH); } @@ -1957,7 +1983,7 @@ ksocknal_autoconnect (ksock_route_t *route) write_lock_irqsave (&ksocknal_data.ksnd_global_lock, flags); for (;;) { - if (!ksocknal_tunables.ksnd_typed_conns) { + if (!*ksocknal_tunables.ksnd_typed_conns) { if ((route->ksnr_connected & (1< n * p) + if (*ksocknal_tunables.ksnd_timeout > n * p) chunk = (chunk * n * p) / - ksocknal_tunables.ksnd_io_timeout; + *ksocknal_tunables.ksnd_timeout; if (chunk == 0) chunk = 1; diff --git a/lnet/klnds/socklnd/socklnd_lib-darwin.c b/lnet/klnds/socklnd/socklnd_lib-darwin.c index ada5b64..7c1bc93 100644 --- a/lnet/klnds/socklnd/socklnd_lib-darwin.c +++ b/lnet/klnds/socklnd/socklnd_lib-darwin.c @@ -37,28 +37,31 @@ #define SOCKNAL_SINGLE_FRAG_RX 1 #endif +#if !CFS_SYSFS_MODULE_PARM +#error "this can't use ksocknal_tunables to get the addresses of the tuning vars" + SYSCTL_DECL(_portals); SYSCTL_NODE (_portals, OID_AUTO, ksocknal, CTLFLAG_RW, 0, "ksocknal_sysctl"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, timeout, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_io_timeout, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_timeout, 0, "timeout"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, eager_ack, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_eager_ack, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_eager_ack, 0, "eager_ack"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, typed, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_typed_conns, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_typed_conns, 0, "typed"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, min_bulk, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_min_bulk, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_min_bulk, 0, "min_bulk"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, buffer_size, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_buffer_size, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_buffer_size, 0, "buffer_size"); SYSCTL_INT(_portals_ksocknal, OID_AUTO, nagle, - CTLTYPE_INT | CTLFLAG_RW , &ksocknal_tunables.ksnd_nagle, + CTLTYPE_INT | CTLFLAG_RW , ksocknal_tunables.ksnd_nagle, 0, "nagle"); cfs_sysctl_table_t ksocknal_top_ctl_table [] = { @@ -72,6 +75,37 @@ cfs_sysctl_table_t ksocknal_top_ctl_table [] = { NULL }; +int +ksocknal_lib_tunables_init () +{ + ksocknal_tunables.ksnd_sysctl = + register_sysctl_table (ksocknal_top_ctl_table, 0); + + if (ksocknal_tunables.ksnd_sysctl == NULL) + return -ENOMEM; + + return 0; +} + +int +ksocknal_lib_tunables_fini () +{ + if (ksocknal_tunables.ksnd_sysctl != NULL) + unregister_sysctl_table (ksocknal_tunables.ksnd_sysctl); +} +#else +int +ksocknal_lib_tunables_init () +{ + return 0; +} + +int +ksocknal_lib_tunables_fini () +{ +} +#endif + static unsigned long ksocknal_mbuf_size = (u_quad_t)SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); struct socket * @@ -665,7 +699,7 @@ ksocknal_lib_setup_sock (struct socket *so) } - if (!ksocknal_tunables.ksnd_nagle) { + if (!*ksocknal_tunables.ksnd_nagle) { option = 1; bzero(&sopt, sizeof sopt); sopt.sopt_dir = SOPT_SET; @@ -679,8 +713,8 @@ ksocknal_lib_setup_sock (struct socket *so) goto out; } } - if (ksocknal_tunables.ksnd_buffer_size > 0) { - option = ksocknal_tunables.ksnd_buffer_size; + if (*ksocknal_tunables.ksnd_buffer_size > 0) { + option = *ksocknal_tunables.ksnd_buffer_size; if (option > ksocknal_mbuf_size) option = ksocknal_mbuf_size; @@ -705,9 +739,9 @@ ksocknal_lib_setup_sock (struct socket *so) } } /* snapshot tunables */ - keep_idle = ksocknal_tunables.ksnd_keepalive_idle; - keep_count = ksocknal_tunables.ksnd_keepalive_count; - keep_intvl = ksocknal_tunables.ksnd_keepalive_intvl; + keep_idle = *ksocknal_tunables.ksnd_keepalive_idle; + keep_count = *ksocknal_tunables.ksnd_keepalive_count; + keep_intvl = *ksocknal_tunables.ksnd_keepalive_intvl; do_keepalive = (keep_idle > 0 && keep_count > 0 && keep_intvl > 0); option = (do_keepalive ? 1 : 0); @@ -799,7 +833,7 @@ ksocknal_lib_connect_sock (struct socket **sockp, int *may_retry, /* Set the socket timeouts, so our connection attempt completes in * finite time */ - tv.tv_sec = ksocknal_tunables.ksnd_io_timeout; + tv.tv_sec = *ksocknal_tunables.ksnd_timeout; tv.tv_usec = 0; bzero(&sopt, sizeof sopt); sopt.sopt_dir = SOPT_SET; @@ -813,7 +847,7 @@ ksocknal_lib_connect_sock (struct socket **sockp, int *may_retry, if (rc != 0) { CFS_NET_EX; CERROR ("Can't set send timeout %d: %d\n", - ksocknal_tunables.ksnd_io_timeout, rc); + *ksocknal_tunables.ksnd_timeout, rc); goto out; } sopt.sopt_level = SOL_SOCKET; @@ -822,7 +856,7 @@ ksocknal_lib_connect_sock (struct socket **sockp, int *may_retry, if (rc != 0) { CFS_NET_EX; CERROR ("Can't set receive timeout %d: %d\n", - ksocknal_tunables.ksnd_io_timeout, rc); + *ksocknal_tunables.ksnd_timeout, rc); goto out; } option = 1; diff --git a/lnet/klnds/socklnd/socklnd_lib-linux.c b/lnet/klnds/socklnd/socklnd_lib-linux.c index 48a813e..3bfcf03 100644 --- a/lnet/klnds/socklnd/socklnd_lib-linux.c +++ b/lnet/klnds/socklnd/socklnd_lib-linux.c @@ -1,65 +1,86 @@ #include "socknal.h" -#ifdef CONFIG_SYSCTL -#define SOCKNAL_SYSCTL 200 - -#define SOCKNAL_SYSCTL_TIMEOUT 1 -#define SOCKNAL_SYSCTL_EAGER_ACK 2 -#define SOCKNAL_SYSCTL_ZERO_COPY 3 -#define SOCKNAL_SYSCTL_TYPED 4 -#define SOCKNAL_SYSCTL_MIN_BULK 5 -#define SOCKNAL_SYSCTL_BUFFER_SIZE 6 -#define SOCKNAL_SYSCTL_NAGLE 7 -#define SOCKNAL_SYSCTL_IRQ_AFFINITY 8 -#define SOCKNAL_SYSCTL_KEEPALIVE_IDLE 9 -#define SOCKNAL_SYSCTL_KEEPALIVE_COUNT 10 -#define SOCKNAL_SYSCTL_KEEPALIVE_INTVL 11 - -static ctl_table ksocknal_ctl_table[] = { - {SOCKNAL_SYSCTL_TIMEOUT, "timeout", - &ksocknal_tunables.ksnd_io_timeout, sizeof (int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_EAGER_ACK, "eager_ack", - &ksocknal_tunables.ksnd_eager_ack, sizeof (int), - 0644, NULL, &proc_dointvec}, +# if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM +static ctl_table ksocknal_ctl_table[12]; + +ctl_table ksocknal_top_ctl_table[] = { + {200, "socknal", NULL, 0, 0555, ksocknal_ctl_table}, + { 0 } +}; + +int +ksocknal_lib_tunables_init () +{ + int i = 0; + int j = 1; + + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "timeout", ksocknal_tunables.ksnd_timeout, + sizeof (int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "eager_ack", ksocknal_tunables.ksnd_eager_ack, + sizeof (int), 0644, NULL, &proc_dointvec}; #if SOCKNAL_ZC - {SOCKNAL_SYSCTL_ZERO_COPY, "zero_copy", - &ksocknal_tunables.ksnd_zc_min_frag, sizeof (int), - 0644, NULL, &proc_dointvec}, + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "zero_copy", ksocknal_tunables.ksnd_zc_min_frag, + sizeof (int), 0644, NULL, &proc_dointvec}; #endif - {SOCKNAL_SYSCTL_TYPED, "typed", - &ksocknal_tunables.ksnd_typed_conns, sizeof (int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_MIN_BULK, "min_bulk", - &ksocknal_tunables.ksnd_min_bulk, sizeof (int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_BUFFER_SIZE, "buffer_size", - &ksocknal_tunables.ksnd_buffer_size, sizeof(int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_NAGLE, "nagle", - &ksocknal_tunables.ksnd_nagle, sizeof(int), - 0644, NULL, &proc_dointvec}, + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "typed", ksocknal_tunables.ksnd_typed_conns, + sizeof (int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "min_bulk", ksocknal_tunables.ksnd_min_bulk, + sizeof (int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "buffer_size", ksocknal_tunables.ksnd_buffer_size, + sizeof(int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "nagle", ksocknal_tunables.ksnd_nagle, + sizeof(int), 0644, NULL, &proc_dointvec}; #if CPU_AFFINITY - {SOCKNAL_SYSCTL_IRQ_AFFINITY, "irq_affinity", - &ksocknal_tunables.ksnd_irq_affinity, sizeof(int), - 0644, NULL, &proc_dointvec}, + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "irq_affinity", ksocknal_tunables.ksnd_irq_affinity, + sizeof(int), 0644, NULL, &proc_dointvec}; #endif - {SOCKNAL_SYSCTL_KEEPALIVE_IDLE, "keepalive_idle", - &ksocknal_tunables.ksnd_keepalive_idle, sizeof(int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_KEEPALIVE_COUNT, "keepalive_count", - &ksocknal_tunables.ksnd_keepalive_count, sizeof(int), - 0644, NULL, &proc_dointvec}, - {SOCKNAL_SYSCTL_KEEPALIVE_INTVL, "keepalive_intvl", - &ksocknal_tunables.ksnd_keepalive_intvl, sizeof(int), - 0644, NULL, &proc_dointvec}, - { 0 } -}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "keepalive_idle", ksocknal_tunables.ksnd_keepalive_idle, + sizeof(int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "keepalive_count", ksocknal_tunables.ksnd_keepalive_count, + sizeof(int), 0644, NULL, &proc_dointvec}; + ksocknal_ctl_table[i++] = (ctl_table) + {j++, "keepalive_intvl", ksocknal_tunables.ksnd_keepalive_intvl, + sizeof(int), 0644, NULL, &proc_dointvec}; + + LASSERT (j == i+1); + LASSERT (i < sizeof(ksocknal_ctl_table)/sizeof(ksocknal_ctl_table[0])); + + ksocknal_tunables.ksnd_sysctl = + register_sysctl_table(ksocknal_top_ctl_table, 0); + + if (ksocknal_tunables.ksnd_sysctl == NULL) + CWARN("Can't setup /proc tunables\n"); + + return 0; +} -ctl_table ksocknal_top_ctl_table[] = { - {SOCKNAL_SYSCTL, "socknal", NULL, 0, 0555, ksocknal_ctl_table}, - { 0 } -}; +void +ksocknal_lib_tunables_fini () +{ + if (ksocknal_tunables.ksnd_sysctl != NULL) + unregister_sysctl_table(ksocknal_tunables.ksnd_sysctl); +} +#else +int +ksocknal_lib_tunables_init () +{ + return 0; +} + +void +ksocknal_lib_tunables_fini () +{ +} #endif void @@ -274,7 +295,7 @@ ksocknal_lib_send_kiov (ksock_conn_t *conn, ksock_tx_t *tx) * or leave them alone. */ #if SOCKNAL_ZC - if (kiov->kiov_len >= ksocknal_tunables.ksnd_zc_min_frag && + if (kiov->kiov_len >= *ksocknal_tunables.ksnd_zc_min_frag && (sock->sk->route_caps & NETIF_F_SG) && (sock->sk->route_caps & (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM))) { struct page *page = kiov->kiov_page; @@ -620,7 +641,7 @@ ksocknal_lib_setup_sock (struct socket *sock) } if (ksocknal_tunables.ksnd_buffer_size > 0) { - option = ksocknal_tunables.ksnd_buffer_size; + option = *ksocknal_tunables.ksnd_buffer_size; set_fs (KERNEL_DS); rc = sock_setsockopt (sock, SOL_SOCKET, SO_SNDBUF, @@ -644,9 +665,9 @@ ksocknal_lib_setup_sock (struct socket *sock) } /* snapshot tunables */ - keep_idle = ksocknal_tunables.ksnd_keepalive_idle; - keep_count = ksocknal_tunables.ksnd_keepalive_count; - keep_intvl = ksocknal_tunables.ksnd_keepalive_intvl; + keep_idle = *ksocknal_tunables.ksnd_keepalive_idle; + keep_count = *ksocknal_tunables.ksnd_keepalive_count; + keep_intvl = *ksocknal_tunables.ksnd_keepalive_intvl; do_keepalive = (keep_idle > 0 && keep_count > 0 && keep_intvl > 0); @@ -749,7 +770,7 @@ ksocknal_lib_connect_sock(struct socket **sockp, int *may_retry, /* Set the socket timeouts, so our connection attempt completes in * finite time */ - tv.tv_sec = ksocknal_tunables.ksnd_io_timeout; + tv.tv_sec = *ksocknal_tunables.ksnd_timeout; tv.tv_usec = 0; set_fs (KERNEL_DS); @@ -758,7 +779,7 @@ ksocknal_lib_connect_sock(struct socket **sockp, int *may_retry, set_fs (oldmm); if (rc != 0) { CERROR ("Can't set send timeout %d: %d\n", - ksocknal_tunables.ksnd_io_timeout, rc); + *ksocknal_tunables.ksnd_timeout, rc); goto failed; } @@ -768,7 +789,7 @@ ksocknal_lib_connect_sock(struct socket **sockp, int *may_retry, set_fs (oldmm); if (rc != 0) { CERROR ("Can't set receive timeout %d: %d\n", - ksocknal_tunables.ksnd_io_timeout, rc); + *ksocknal_tunables.ksnd_timeout, rc); goto failed; } diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c new file mode 100644 index 0000000..b8ade09 --- /dev/null +++ b/lnet/klnds/socklnd/socklnd_modparams.c @@ -0,0 +1,88 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Eric Barton + * + * Portals is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Portals is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Portals; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "socknal.h" + +static int timeout = SOCKNAL_IO_TIMEOUT; +CFS_MODULE_PARM(timeout, "i", int, 0644, + "dead socket timeout (seconds)"); + +static int eager_ack = SOCKNAL_EAGER_ACK; +CFS_MODULE_PARM(eager_ack, "i", int, 0644, + "send tcp ack packets eagerly"); + +static int typed_conns = SOCKNAL_TYPED_CONNS; +CFS_MODULE_PARM(typed_conns, "i", int, 0644, + "use different sockets for bulk"); + +static int min_bulk = SOCKNAL_MIN_BULK; +CFS_MODULE_PARM(min_bulk, "i", int, 0644, + "smallest 'large' message"); + +static int buffer_size = SOCKNAL_BUFFER_SIZE; +CFS_MODULE_PARM(buffer_size, "i", int, 0644, + "socket buffer size"); + +static int nagle = SOCKNAL_NAGLE; +CFS_MODULE_PARM(nagle, "i", int, 0644, + "enable NAGLE?"); + +static int keepalive_idle = SOCKNAL_KEEPALIVE_IDLE; +CFS_MODULE_PARM(keepalive_idle, "i", int, 0644, + "# idle seconds before probe"); + +static int keepalive_count = SOCKNAL_KEEPALIVE_COUNT; +CFS_MODULE_PARM(keepalive_count, "i", int, 0644, + "# missed probes == dead"); + +static int keepalive_intvl = SOCKNAL_KEEPALIVE_INTVL; +CFS_MODULE_PARM(keepalive_intvl, "i", int, 0644, + "seconds between probes"); + +#if CPU_AFFINITY +static int irq_affinity = SOCKNAL_IRQ_AFFINITY; +CFS_MODULE_PARM(irq_affinity, "i", int, 0644, + "enable IRQ affinity"); +#endif + +#if SOCKNAL_ZC +static unsigned int zc_min_frag = SOCKNAL_ZC_MIN_FRAG; +CFS_MODULE_PARM(zc_min_frag, "i", int, 0644, + "minimum fragment to zero copy"); +#endif + +ksock_tunables_t ksocknal_tunables = { + .ksnd_timeout = &timeout, + .ksnd_eager_ack = &eager_ack, + .ksnd_typed_conns = &typed_conns, + .ksnd_min_bulk = &min_bulk, + .ksnd_buffer_size = &buffer_size, + .ksnd_nagle = &nagle, + .ksnd_keepalive_idle = &keepalive_idle, + .ksnd_keepalive_count = &keepalive_count, + .ksnd_keepalive_intvl = &keepalive_intvl, +#if SOCKNAL_ZC + .ksnd_zc_min_frag = &zc_min_frag, +#endif +#if CPU_AFFINITY + .ksnd_irq_affinity = &irq_affinity, +#endif +}; + diff --git a/lnet/klnds/viblnd/viblnd.c b/lnet/klnds/viblnd/viblnd.c index 69558c6..b960020 100644 --- a/lnet/klnds/viblnd/viblnd.c +++ b/lnet/klnds/viblnd/viblnd.c @@ -25,7 +25,6 @@ #include "vibnal.h" ptl_nal_t kibnal_nal = { - .nal_name = "vib", .nal_type = VIBNAL, .nal_startup = kibnal_startup, .nal_shutdown = kibnal_shutdown, @@ -1898,11 +1897,7 @@ kibnal_module_init (void) /* Initialise dynamic tunables to defaults once only */ kibnal_tunables.kib_io_timeout = IBNAL_IO_TIMEOUT; - rc = ptl_register_nal(&kibnal_nal); - if (rc != PTL_OK) { - CERROR("Can't register IBNAL: %d\n", rc); - return (-ENOMEM); /* or something... */ - } + ptl_register_nal(&kibnal_nal); #ifdef CONFIG_SYSCTL /* Press on regardless even if registering sysctl doesn't work */ diff --git a/lnet/klnds/viblnd/viblnd.h b/lnet/klnds/viblnd/viblnd.h index eb9aef5..5d0796e 100644 --- a/lnet/klnds/viblnd/viblnd.h +++ b/lnet/klnds/viblnd/viblnd.h @@ -572,6 +572,8 @@ kibnal_page2phys (struct page *p) # if CONFIG_X86 && CONFIG_HIGHMEM4G /* truncation to void* doesn't matter if 0 <= physmem < 4G * so allow x86 with 32 bit phys addrs */ +# elif CONFIG_IA64 + /* OK anyway on 64-bit arch */ # else # error "Can't support HIGHMEM when vv_scatgat_t::v_address is void *" # endif diff --git a/lnet/libcfs/nidstrings.c b/lnet/libcfs/nidstrings.c index 6562f86..6f78cd9 100644 --- a/lnet/libcfs/nidstrings.c +++ b/lnet/libcfs/nidstrings.c @@ -64,37 +64,252 @@ libcfs_next_nidstring (void) return str; } -char *libcfs_nid2str(ptl_nid_t nid) +#if !CRAY_PORTALS +static void libcfs_lo_addr2str(__u32 addr, char *str); +static int libcfs_lo_str2addr(char *str, int nob, __u32 *addr); +static void libcfs_ip_addr2str(__u32 addr, char *str); +static int libcfs_ip_str2addr(char *str, int nob, __u32 *addr); +static void libcfs_num_addr2str(__u32 addr, char *str); +static int libcfs_num_str2addr(char *str, int nob, __u32 *addr); + +struct nettype { + int type; + char *name; + void (*addr2str)(__u32 addr, char *str); + int (*str2addr)(char *str, int nob, __u32 *addr); +}; + +static struct nettype libcfs_nettypes[] = { + {LONAL, "lo", libcfs_lo_addr2str, libcfs_lo_str2addr}, + {SOCKNAL, "tcp", libcfs_ip_addr2str, libcfs_ip_str2addr}, + {OPENIBNAL, "openib", libcfs_ip_addr2str, libcfs_ip_str2addr}, + {IIBNAL, "iib", libcfs_ip_addr2str, libcfs_ip_str2addr}, + {VIBNAL, "vib", libcfs_ip_addr2str, libcfs_ip_str2addr}, + {RANAL, "ra", libcfs_ip_addr2str, libcfs_ip_str2addr}, + {QSWNAL, "elan", libcfs_num_addr2str, libcfs_num_str2addr}, + {GMNAL, "gm", libcfs_num_addr2str, libcfs_num_str2addr}, +}; + +const int libcfs_nnettypes = sizeof(libcfs_nettypes)/sizeof(libcfs_nettypes[0]); + +void +libcfs_lo_addr2str(__u32 addr, char *str) +{ + /* don't print anything */ +} + +int +libcfs_lo_str2addr(char *str, int nob, __u32 *addr) +{ + if (nob != 0) /* expecting the empty string */ + return 0; + + *addr = 0; + return 1; +} + +void +libcfs_ip_addr2str(__u32 addr, char *str) +{ + snprintf(str, PTL_NALFMT_SIZE, "%u.%u.%u.%u", HIPQUAD(addr)); +} + +int +libcfs_ip_str2addr(char *str, int nob, __u32 *addr) { - __u32 hi = (__u32)(nid>>32); - __u32 lo = (__u32)nid; - char *str = libcfs_next_nidstring(); + int a; + int b; + int c; + int d; + int n; + + if (sscanf(str, "%u.%u.%u.%u%n", &a, &b, &c, &d, &n) < 4 || + n != nob) + return 0; + + if ((a & ~0xff) != 0 || (b & ~0xff) != 0 || + (c & ~0xff) != 0 || (d & ~0xff) != 0) + return 0; + + *addr = ((a<<24)|(b<<16)|(c<<8)|d); + return 1; +} + +void +libcfs_num_addr2str(__u32 addr, char *str) +{ + snprintf(str, PTL_NALFMT_SIZE, "%u", addr); +} + +int +libcfs_num_str2addr(char *str, int nob, __u32 *addr) +{ + __u32 a; + int n; + + if (sscanf(str, "%u%n", &a, &n) < 1 || + n != nob) + return 0; + + *addr = a; + return 1; +} + +struct nettype * +libcfs_get_nettype(int type) +{ + int i; + + for (i = 0; i < libcfs_nnettypes; i++) + if (type == libcfs_nettypes[i].type) + return &libcfs_nettypes[i]; + + return NULL; +} + +int +libcfs_isknown_nettype(int type) +{ + return libcfs_get_nettype(type) != NULL; +} + +char * +libcfs_nettype2str(int type) +{ + char *str; + struct nettype *nt = libcfs_get_nettype(type); + + if (nt != NULL) + return nt->name; + + str = libcfs_next_nidstring(); + snprintf(str, PTL_NALFMT_SIZE, "?%u?", type); + return str; +} + +char * +libcfs_nid2str(ptl_nid_t nid) +{ + char *str = libcfs_next_nidstring(); + __u32 lo = (__u32)nid; + __u32 hi = (__u32)(nid>>32); + int nnum = hi & 0xffff; + int ntype = (hi >> 16) & 0xffff; + struct nettype *nettype = libcfs_get_nettype(ntype); + int nob; if (nid == PTL_NID_ANY) { snprintf(str, PTL_NALFMT_SIZE, "%s", "PTL_NID_ANY"); - return str; + } else if (nettype == NULL) { + snprintf(str, PTL_NALFMT_SIZE, "%x@%u.%u", lo, nnum, ntype); + } else { + nettype->addr2str(lo, str); + nob = strlen(str); + if (nnum == 0) + snprintf(str + nob, PTL_NALFMT_SIZE - nob, + "@%s", nettype->name); + else + snprintf(str + nob, PTL_NALFMT_SIZE - nob, + "@%s%u", nettype->name, nnum); } -#if !CRAY_PORTALS - if ((lo & 0xffff) != 0) { - /* probable IP address */ - if (hi != 0) - snprintf(str, PTL_NALFMT_SIZE, "%u:%u.%u.%u.%u", - hi, HIPQUAD(lo)); - else - snprintf(str, PTL_NALFMT_SIZE, "%u.%u.%u.%u", - HIPQUAD(lo)); - } else if (hi != 0) - snprintf(str, PTL_NALFMT_SIZE, "%u:%u", hi, lo); - else - snprintf(str, PTL_NALFMT_SIZE, "%u", lo); -#else - snprintf(str, PTL_NALFMT_SIZE, "%llx", (long long)nid); -#endif return str; } -char *libcfs_id2str(ptl_process_id_t id) +ptl_nid_t +libcfs_str2nid(char *str) +{ + char *sep = strchr(str, '@'); + struct nettype *nettype; + int nob; + int net; + __u32 addr; + int i; + + if (sep == NULL) { + sep = str + strlen(str); + net = 0; + nettype = libcfs_get_nettype(SOCKNAL); + LASSERT (nettype != NULL); + } else { + for (i = 0; i < libcfs_nnettypes; i++) { + nettype = &libcfs_nettypes[i]; + + if (!strncmp(sep + 1, nettype->name, + strlen(nettype->name))) + break; + } + if (i == libcfs_nnettypes) + return PTL_NID_ANY; + + nob = strlen(nettype->name); + + if (strlen(sep + 1) == nob) + net = 0; + else if (nettype->type == LONAL || /* net number not allowed */ + sscanf(sep + 1 + nob, "%u%n", &net, &i) < 1 || + i != strlen(sep + 1 + nob) || + (net & ~0xffff) != 0) + return PTL_NID_ANY; + } + + if (!nettype->str2addr(str, sep - str, &addr)) + return PTL_NID_ANY; + + return (((__u64)((nettype->type<<16)|net))<<32)|addr; +} +#else /* CRAY_PORTALS */ +int +libcfs_isknown_nettype(int type) +{ + return 1; +} + +char * +libcfs_nettype2str(int type) +{ + return "cray"; +} + +char * +libcfs_nid2str(ptl_nid_t nid) +{ + char *str = libcfs_next_nidstring(); + + snprintf(str, PTL_NALFMT_SIZE, "%llx", (unsigned long long)nid); +} + +ptl_nid_t +libcfs_str2nid(char *str) +{ + long long nid; + long long mask; + int n; + + if (sscanf(str,"%llx%n", &nid, &n) >= 1 && + n == strlen(str)) + goto out; + + if (sscanf(str,"%llu%n", &nid, &n) >= 1 && + n = strlen(str)) + goto out; + + return PTL_NID_ANY; + + out: + /* overflow check in case ptl_nid_t smaller than __u64 */ + mask = 0; + mask = (~mask)<<(sizeof(nid)*8); + + if ((nid & mask) != 0) + return PTL_NID_ANY; + + return nid; +} +#endif + +char * +libcfs_id2str(ptl_process_id_t id) { char *str = libcfs_nid2str(id.nid); int len = strlen(str); @@ -103,5 +318,7 @@ char *libcfs_id2str(ptl_process_id_t id) return str; } +EXPORT_SYMBOL(libcfs_isknown_nettype); +EXPORT_SYMBOL(libcfs_nettype2str); EXPORT_SYMBOL(libcfs_nid2str); EXPORT_SYMBOL(libcfs_id2str); diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 0b8f121..4b0a891 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -143,24 +143,6 @@ static void ptl_mutex_exit (void) ptl_nal_t * -ptl_find_nal_by_name (char *name) -{ - ptl_nal_t *nal; - struct list_head *tmp; - - /* holding mutex */ - list_for_each (tmp, &ptl_nal_table) { - nal = list_entry(tmp, ptl_nal_t, nal_list); - - if (!strcmp (nal->nal_name, name)) - return nal; - } - - return NULL; -} - - -ptl_nal_t * ptl_find_nal_by_type (int type) { ptl_nal_t *nal; @@ -178,27 +160,19 @@ ptl_find_nal_by_type (int type) } -int +void ptl_register_nal (ptl_nal_t *nal) { - int rc; - ptl_mutex_enter(); LASSERT (ptl_init); - - if (ptl_find_nal_by_name(nal->nal_name) != NULL || - ptl_find_nal_by_type(nal->nal_type) != NULL) { - rc = PTL_IFACE_DUP; - } else { - list_add (&nal->nal_list, &ptl_nal_table); - - nal->nal_refcount = 0; - rc = PTL_OK; - } + LASSERT (libcfs_isknown_nettype(nal->nal_type)); + LASSERT (ptl_find_nal_by_type(nal->nal_type) == NULL); + + list_add (&nal->nal_list, &ptl_nal_table); + nal->nal_refcount = 0; ptl_mutex_exit(); - return (rc); } void @@ -208,7 +182,6 @@ ptl_unregister_nal (ptl_nal_t *nal) LASSERT (ptl_init); LASSERT (ptl_find_nal_by_type(nal->nal_type) == nal); - LASSERT (ptl_find_nal_by_name(nal->nal_name) == nal); LASSERT (nal->nal_refcount == 0); list_del (&nal->nal_list); @@ -615,19 +588,20 @@ ptl_startup_nalnis (void) PORTAL_ALLOC(ni, sizeof(*ni)); if (ni == NULL) { - CERROR("Can't allocate NI for %s\n", - nal->nal_name); + CERROR("Can't allocate NI for %s NAL\n", + libcfs_nettype2str(nal->nal_type)); rc = PTL_FAIL; break; } ni->ni_nal = nal; + // ni->ni_nid = (nal->nal_type << 16) nal->nal_refcount++; rc = (nal->nal_startup)(ni, &interface); if (rc != PTL_OK) { CERROR("Error %d staring up NI %s\n", - rc, nal->nal_name); + rc, libcfs_nettype2str(nal->nal_type)); PORTAL_FREE(ni, sizeof(*ni)); nal->nal_refcount--; break; diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index b307df1..f47b033 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -200,11 +200,8 @@ PtlMDAttach(ptl_handle_me_t meh, ptl_md_t umd, unsigned long flags; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_NI_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if ((umd.options & (PTL_MD_KIOV | PTL_MD_IOVEC)) != 0 && umd.length > PTL_MD_MAX_IOV) /* too many fragments */ @@ -248,11 +245,8 @@ PtlMDBind(ptl_handle_ni_t nih, ptl_md_t umd, unsigned long flags; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_NI_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if ((umd.options & (PTL_MD_KIOV | PTL_MD_IOVEC)) != 0 && umd.length > PTL_MD_MAX_IOV) /* too many fragments */ @@ -286,11 +280,8 @@ PtlMDUnlink (ptl_handle_md_t mdh) ptl_libmd_t *md; unsigned long flags; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_MD_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); @@ -333,11 +324,8 @@ PtlMDUpdate(ptl_handle_md_t mdh, unsigned long flags; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_MD_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index f521ed5..2891222 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -38,11 +38,8 @@ PtlMEAttach(ptl_handle_ni_t interface, ptl_me_t *me; unsigned long flags; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_NI_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if (portal >= ptl_apini.apini_nportals) return PTL_PT_INDEX_INVALID; @@ -85,11 +82,8 @@ PtlMEInsert(ptl_handle_me_t current_meh, ptl_me_t *new_me; unsigned long flags; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_ME_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); new_me = ptl_me_alloc(); if (new_me == NULL) @@ -132,11 +126,8 @@ PtlMEUnlink(ptl_handle_me_t meh) ptl_me_t *me; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_ME_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 44b86e6..7a708b6 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -173,11 +173,8 @@ PtlFailNid (ptl_handle_ni_t interface, struct list_head *next; struct list_head cull; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_HANDLE_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if (threshold != 0) { /* Adding a new entry */ @@ -1416,17 +1413,20 @@ do_ptl_parse(ptl_ni_t *ni, ptl_hdr_t *hdr, void *private, int loopback) ptl_ni_t * ptl_nid2ni (ptl_nid_t nid) { + struct list_head *tmp; + ptl_ni_t *ni; + /* Called holding PTL_LOCK */ - if (list_empty(&ptl_apini.apini_nis)) - return NULL; - - if (ptl_apini.apini_nis.next != ptl_apini.apini_nis.prev) { - CERROR ("Can't decide which NI\n"); - return NULL; + list_for_each (tmp, &ptl_apini.apini_nis) { + ni = list_entry(tmp, ptl_ni_t, ni_list); + + /* network type & number match in target NID and ni's NID */ + if (((ni->ni_nid ^ nid)>>32) == 0) + return ni; } - - return list_entry(ptl_apini.apini_nis.next, ptl_ni_t, ni_list); + + return NULL; } ptl_err_t @@ -1442,11 +1442,8 @@ PtlPut(ptl_handle_md_t mdh, ptl_ack_req_t ack, unsigned long flags; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_MD_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if (!list_empty (&ptl_apini.apini_test_peers) && /* normally we don't */ fail_peer (target.nid, 1)) /* shall we now? */ @@ -1616,11 +1613,8 @@ PtlGet(ptl_handle_md_t mdh, ptl_process_id_t target, unsigned long flags; int rc; - if (!ptl_init) - return PTL_NO_INIT; - - if (ptl_apini.apini_refcount == 0) - return PTL_MD_INVALID; + LASSERT (ptl_init); + LASSERT (ptl_apini.apini_refcount > 0); if (!list_empty (&ptl_apini.apini_test_peers) && /* normally we don't */ fail_peer (target.nid, 1)) /* shall we now? */ diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c index 7251a47..3aec8b8 100644 --- a/lnet/ulnds/procapi.c +++ b/lnet/ulnds/procapi.c @@ -71,7 +71,6 @@ extern ptl_err_t tcpnal_recv(ptl_ni_t *ni, void *private, ptl_msg_t *cookie, size_t offset, size_t mlen, size_t rlen); ptl_nal_t tcpnal_nal = { - .nal_name = "tcp", .nal_type = SOCKNAL, .nal_startup = procbridge_startup, .nal_shutdown = procbridge_shutdown, diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c index 7251a47..3aec8b8 100644 --- a/lnet/ulnds/socklnd/procapi.c +++ b/lnet/ulnds/socklnd/procapi.c @@ -71,7 +71,6 @@ extern ptl_err_t tcpnal_recv(ptl_ni_t *ni, void *private, ptl_msg_t *cookie, size_t offset, size_t mlen, size_t rlen); ptl_nal_t tcpnal_nal = { - .nal_name = "tcp", .nal_type = SOCKNAL, .nal_startup = procbridge_startup, .nal_shutdown = procbridge_shutdown,