#include "qswnal.h"
-ptl_handle_ni_t kqswnal_ni;
nal_t kqswnal_api;
kqswnal_data_t kqswnal_data;
+ptl_handle_ni_t kqswnal_ni;
+kqswnal_tunables_t kqswnal_tunables;
kpr_nal_interface_t kqswnal_router_interface = {
kprni_nalid: QSWNAL,
static ctl_table kqswnal_ctl_table[] = {
{QSWNAL_SYSCTL_OPTIMIZED_GETS, "optimized_gets",
- &kqswnal_data.kqn_optimized_gets, sizeof (int),
- 0644, NULL, &proc_dointvec},
- {QSWNAL_SYSCTL_COPY_SMALL_FWD, "copy_small_fwd",
- &kqswnal_data.kqn_copy_small_fwd, sizeof (int),
+ &kqswnal_tunables.kqn_optimized_gets, sizeof (int),
0644, NULL, &proc_dointvec},
{0}
};
}
static int
-kqswnal_shutdown(nal_t *nal, int ni)
-{
- CDEBUG (D_NET, "shutdown\n");
-
- LASSERT (nal == &kqswnal_api);
- return (0);
-}
-
-static int
kqswnal_yield(nal_t *nal, unsigned long *flags, int milliseconds)
{
/* NB called holding statelock */
return (milliseconds);
}
-static nal_t *
-kqswnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size,
- ptl_pid_t requested_pid)
-{
- ptl_nid_t mynid = kqswnal_elanid2nid (kqswnal_data.kqn_elanid);
- int nnids = kqswnal_data.kqn_nnodes;
-
- CDEBUG(D_NET, "calling lib_init with nid "LPX64" of %d\n", mynid, nnids);
-
- lib_init(&kqswnal_lib, mynid, 0, nnids, ptl_size, ac_size);
-
- return (&kqswnal_api);
-}
-
int
kqswnal_get_tx_desc (struct portals_cfg *pcfg)
{
}
}
-void __exit
-kqswnal_finalise (void)
+static void
+kqswnal_shutdown(nal_t *nal)
{
unsigned long flags;
- int do_ptl_fini = 0;
+ int do_lib_fini = 0;
+
+ /* NB The first ref was this module! */
+ if (nal->nal_refct != 0) {
+ PORTAL_MODULE_UNUSE;
+ return;
+ }
+
+ CDEBUG (D_NET, "shutdown\n");
+ LASSERT (nal == &kqswnal_api);
switch (kqswnal_data.kqn_init)
{
LASSERT (0);
case KQN_INIT_ALL:
-#if CONFIG_SYSCTL
- if (kqswnal_data.kqn_sysctl != NULL)
- unregister_sysctl_table (kqswnal_data.kqn_sysctl);
-#endif
- PORTAL_SYMBOL_UNREGISTER (kqswnal_ni);
- kportal_nal_unregister(QSWNAL);
+ libcfs_nal_cmd_unregister(QSWNAL);
/* fall through */
- case KQN_INIT_PTL:
- do_ptl_fini = 1;
+ case KQN_INIT_LIB:
+ do_lib_fini = 1;
/* fall through */
case KQN_INIT_DATA:
kpr_deregister (&kqswnal_data.kqn_router);
- if (do_ptl_fini) {
- PtlNIFini (kqswnal_ni);
+ if (do_lib_fini)
lib_fini (&kqswnal_lib);
- }
/**********************************************************************/
/* Unmap message buffers and free all descriptors and buffers
}
static int __init
-kqswnal_initialise (void)
+kqswnal_startup (nal_t *nal, ptl_pid_t requested_pid,
+ ptl_ni_limits_t *requested_limits,
+ ptl_ni_limits_t *actual_limits)
{
#if MULTIRAIL_EKC
EP_RAILMASK all_rails = EP_RAILMASK_ALL;
int rc;
int i;
int elan_page_idx;
+ ptl_process_id_t my_process_id;
int pkmem = atomic_read(&portal_kmemory);
+ if (nal->nal_refct != 0) {
+ if (actual_limits != NULL)
+ *actual_limits = kqswnal_lib.ni.actual_limits;
+ /* This module got the first ref */
+ PORTAL_MODULE_USE;
+ return (PTL_OK);
+ }
+
LASSERT (kqswnal_data.kqn_init == KQN_INIT_NOTHING);
CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory));
- kqswnal_api.forward = kqswnal_forward;
- kqswnal_api.shutdown = kqswnal_shutdown;
- kqswnal_api.yield = kqswnal_yield;
- kqswnal_api.validate = NULL; /* our api validate is a NOOP */
- kqswnal_api.lock = kqswnal_lock;
- kqswnal_api.unlock = kqswnal_unlock;
- kqswnal_api.nal_data = &kqswnal_data;
-
- kqswnal_lib.nal_data = &kqswnal_data;
-
memset(&kqswnal_rpc_success, 0, sizeof(kqswnal_rpc_success));
memset(&kqswnal_rpc_failed, 0, sizeof(kqswnal_rpc_failed));
#if MULTIRAIL_EKC
/* ensure all pointers NULL etc */
memset (&kqswnal_data, 0, sizeof (kqswnal_data));
- kqswnal_data.kqn_optimized_gets = KQSW_OPTIMIZED_GETS;
- kqswnal_data.kqn_copy_small_fwd = KQSW_COPY_SMALL_FWD;
-
kqswnal_data.kqn_cb = &kqswnal_lib;
INIT_LIST_HEAD (&kqswnal_data.kqn_idletxds);
/* pointers/lists/locks initialised */
kqswnal_data.kqn_init = KQN_INIT_DATA;
-
+
#if MULTIRAIL_EKC
kqswnal_data.kqn_ep = ep_system();
if (kqswnal_data.kqn_ep == NULL) {
CERROR("Can't initialise EKC\n");
- return (-ENODEV);
+ kqswnal_shutdown(&kqswnal_api);
+ return (PTL_IFACE_INVALID);
}
if (ep_waitfor_nodeid(kqswnal_data.kqn_ep) == ELAN_INVALID_NODE) {
CERROR("Can't get elan ID\n");
- kqswnal_finalise();
- return (-ENODEV);
+ kqswnal_shutdown(&kqswnal_api);
+ return (PTL_IFACE_INVALID);
}
#else
/**********************************************************************/
if (kqswnal_data.kqn_ep == NULL)
{
CERROR ("Can't get elan device 0\n");
- return (-ENODEV);
+ kqswnal_shutdown(&kqswnal_api);
+ return (PTL_IFACE_INVALID);
}
#endif
if (kqswnal_data.kqn_eptx == NULL)
{
CERROR ("Can't allocate transmitter\n");
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
/**********************************************************************/
if (kqswnal_data.kqn_eprx_small == NULL)
{
CERROR ("Can't install small msg receiver\n");
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
kqswnal_data.kqn_eprx_large = ep_alloc_rcvr (kqswnal_data.kqn_ep,
if (kqswnal_data.kqn_eprx_large == NULL)
{
CERROR ("Can't install large msg receiver\n");
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
/**********************************************************************/
EP_PERM_WRITE);
if (kqswnal_data.kqn_ep_tx_nmh == NULL) {
CERROR("Can't reserve tx dma space\n");
- kqswnal_finalise();
- return (-ENOMEM);
+ kqswnal_shutdown(&kqswnal_api);
+ return (PTL_NO_SPACE);
}
#else
dmareq.Waitfn = DDI_DMA_SLEEP;
if (rc != DDI_SUCCESS)
{
CERROR ("Can't reserve rx dma space\n");
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
#endif
/**********************************************************************/
EP_PERM_WRITE);
if (kqswnal_data.kqn_ep_tx_nmh == NULL) {
CERROR("Can't reserve rx dma space\n");
- kqswnal_finalise();
- return (-ENOMEM);
+ kqswnal_shutdown(&kqswnal_api);
+ return (PTL_NO_SPACE);
}
#else
dmareq.Waitfn = DDI_DMA_SLEEP;
if (rc != DDI_SUCCESS)
{
CERROR ("Can't reserve rx dma space\n");
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
#endif
/**********************************************************************/
sizeof(kqswnal_tx_t) * (KQSW_NTXMSGS + KQSW_NNBLK_TXMSGS));
if (kqswnal_data.kqn_txds == NULL)
{
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
/* clear flags, null pointers etc */
PORTAL_ALLOC (ktx->ktx_buffer, KQSW_TX_BUFFER_SIZE);
if (ktx->ktx_buffer == NULL)
{
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
/* Map pre-allocated buffer NOW, to save latency on transmit */
sizeof (kqswnal_rx_t) * (KQSW_NRXMSGS_SMALL + KQSW_NRXMSGS_LARGE));
if (kqswnal_data.kqn_rxds == NULL)
{
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
memset(kqswnal_data.kqn_rxds, 0, /* clear flags, null pointers etc */
struct page *page = alloc_page(GFP_KERNEL);
if (page == NULL) {
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_NO_SPACE);
}
krx->krx_kiov[j].kiov_page = page;
/**********************************************************************/
/* Network interface ready to initialise */
- rc = PtlNIInit(kqswnal_init, 32, 4, 0, &kqswnal_ni);
- if (rc != 0)
+ my_process_id.nid = kqswnal_elanid2nid(kqswnal_data.kqn_elanid);
+ my_process_id.pid = 0;
+
+ rc = lib_init(&kqswnal_lib, my_process_id,
+ requested_limits, actual_limits);
+ if (rc != PTL_OK)
{
- CERROR ("PtlNIInit failed %d\n", rc);
- kqswnal_finalise ();
- return (-ENOMEM);
+ CERROR ("lib_init failed %d\n", rc);
+ kqswnal_shutdown (&kqswnal_api);
+ return (rc);
}
- kqswnal_data.kqn_init = KQN_INIT_PTL;
+ kqswnal_data.kqn_init = KQN_INIT_LIB;
/**********************************************************************/
/* Queue receives, now that it's OK to run their completion callbacks */
if (rc != EP_SUCCESS)
{
CERROR ("failed ep_queue_receive %d\n", rc);
- kqswnal_finalise ();
- return (-ENOMEM);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_FAIL);
}
}
if (rc != 0)
{
CERROR ("failed to spawn scheduling thread: %d\n", rc);
- kqswnal_finalise ();
- return (rc);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_FAIL);
}
}
rc = kpr_register (&kqswnal_data.kqn_router, &kqswnal_router_interface);
CDEBUG(D_NET, "Can't initialise routing interface (rc = %d): not routing\n",rc);
- rc = kportal_nal_register (QSWNAL, &kqswnal_cmd, NULL);
+ rc = libcfs_nal_cmd_register (QSWNAL, &kqswnal_cmd, NULL);
if (rc != 0) {
CERROR ("Can't initialise command interface (rc = %d)\n", rc);
- kqswnal_finalise ();
- return (rc);
+ kqswnal_shutdown (&kqswnal_api);
+ return (PTL_FAIL);
}
-#if CONFIG_SYSCTL
- /* Press on regardless even if registering sysctl doesn't work */
- kqswnal_data.kqn_sysctl = register_sysctl_table (kqswnal_top_ctl_table, 0);
-#endif
-
- PORTAL_SYMBOL_REGISTER(kqswnal_ni);
kqswnal_data.kqn_init = KQN_INIT_ALL;
printk(KERN_INFO "Lustre: Routing QSW NAL loaded on node %d of %d "
kpr_routing (&kqswnal_data.kqn_router) ? "enabled" : "disabled",
pkmem);
- return (0);
+ return (PTL_OK);
}
+void __exit
+kqswnal_finalise (void)
+{
+#if CONFIG_SYSCTL
+ if (kqswnal_tunables.kqn_sysctl != NULL)
+ unregister_sysctl_table (kqswnal_tunables.kqn_sysctl);
+#endif
+ PtlNIFini(kqswnal_ni);
+
+ ptl_unregister_nal(QSWNAL);
+}
+
+static int __init
+kqswnal_initialise (void)
+{
+ int rc;
+
+ kqswnal_api.startup = kqswnal_startup;
+ kqswnal_api.shutdown = kqswnal_shutdown;
+ kqswnal_api.forward = kqswnal_forward;
+ kqswnal_api.yield = kqswnal_yield;
+ kqswnal_api.lock = kqswnal_lock;
+ kqswnal_api.unlock = kqswnal_unlock;
+ kqswnal_api.nal_data = &kqswnal_data;
+
+ kqswnal_lib.nal_data = &kqswnal_data;
+
+ /* Initialise dynamic tunables to defaults once only */
+ kqswnal_tunables.kqn_optimized_gets = KQSW_OPTIMIZED_GETS;
+
+ rc = ptl_register_nal(QSWNAL, &kqswnal_api);
+ if (rc != PTL_OK) {
+ CERROR("Can't register QSWNAL: %d\n", rc);
+ return (-ENOMEM); /* or something... */
+ }
+
+ /* Pure gateways, and the workaround for 'EKC blocks forever until
+ * the service is active' want the NAL started up at module load
+ * time... */
+ rc = PtlNIInit(QSWNAL, 0, NULL, NULL, &kqswnal_ni);
+ if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
+ ptl_unregister_nal(QSWNAL);
+ return (-ENODEV);
+ }
+
+#if CONFIG_SYSCTL
+ /* Press on regardless even if registering sysctl doesn't work */
+ kqswnal_tunables.kqn_sysctl =
+ register_sysctl_table (kqswnal_top_ctl_table, 0);
+#endif
+ return (0);
+}
MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
MODULE_DESCRIPTION("Kernel Quadrics/Elan NAL v1.01");
module_init (kqswnal_initialise);
module_exit (kqswnal_finalise);
-
-EXPORT_SYMBOL (kqswnal_ni);