lnet/klnds/gmlnd/gmnal.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Based on ksocknal and qswnal
   5  *
   6  * Copyright (C) 2002 Cluster File Systems, Inc.
   7  *   Author: Robert Read  <rread@datarithm.net>
   8  *
   9  *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
  10  *
  11  *   Portals is free software; you can redistribute it and/or
  12  *   modify it under the terms of version 2 of the GNU General Public
  13  *   License as published by the Free Software Foundation.
  14  *
  15  *   Portals is distributed in the hope that it will be useful,
  16  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  *   GNU General Public License for more details.
  19  *
  20  *   You should have received a copy of the GNU General Public License
  21  *   along with Portals; if not, write to the Free Software
  22  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23  */
  24
  25 #include "gmnal.h"
  26
  27 ptl_handle_ni_t kgmnal_ni;
  28 nal_t  kgmnal_api;
  29
  30 kgmnal_data_t kgmnal_data;
  31 int gmnal_debug = 0;
  32
  33 kpr_nal_interface_t kqswnal_router_interface = {
  34         kprni_nalid:        GMNAL,
  35         kprni_arg:        NULL,
  36         kprni_fwd:          kgmnal_fwd_packet,
  37 };
  38
  39 static int kgmnal_forward(nal_t   *nal,
  40                           int     id,
  41                           void    *args,  size_t args_len,
  42                           void    *ret,   size_t ret_len)
  43 {
  44         kgmnal_data_t *k = nal->nal_data;
  45         nal_cb_t      *nal_cb = k->kgm_cb;
  46
  47         LASSERT (nal == &kgmnal_api);
  48         LASSERT (k == &kgmnal_data);
  49         LASSERT (nal_cb == &kgmnal_lib);
  50
  51         lib_dispatch(nal_cb, k, id, args, ret); /* nal needs k */
  52         return PTL_OK;
  53 }
  54
  55 static void kgmnal_lock(nal_t *nal, unsigned long *flags)
  56 {
  57         kgmnal_data_t *k = nal->nal_data;
  58         nal_cb_t      *nal_cb = k->kgm_cb;
  59
  60
  61         LASSERT (nal == &kgmnal_api);
  62         LASSERT (k == &kgmnal_data);
  63         LASSERT (nal_cb == &kgmnal_lib);
  64
  65         nal_cb->cb_cli(nal_cb,flags);
  66 }
  67
  68 static void kgmnal_unlock(nal_t *nal, unsigned long *flags)
  69 {
  70         kgmnal_data_t *k = nal->nal_data;
  71         nal_cb_t      *nal_cb = k->kgm_cb;
  72
  73
  74         LASSERT (nal == &kgmnal_api);
  75         LASSERT (k == &kgmnal_data);
  76         LASSERT (nal_cb == &kgmnal_lib);
  77
  78         nal_cb->cb_sti(nal_cb,flags);
  79 }
  80
  81 static int kgmnal_shutdown(nal_t *nal, int ni)
  82 {
  83         LASSERT (nal == &kgmnal_api);
  84         return 0;
  85 }
  86
  87 static void kgmnal_yield( nal_t *nal )
  88 {
  89         LASSERT (nal == &kgmnal_api);
  90
  91         if (current->need_resched)
  92                 schedule();
  93         return;
  94 }
  95
  96 kgmnal_rx_t *kgm_add_recv(kgmnal_data_t *data,int ndx)
  97 {
  98         kgmnal_rx_t *conn;
  99
 100         PORTAL_ALLOC(conn, sizeof(kgmnal_rx_t));
 101         /* Check for out of mem here */
 102         if (conn==NULL) {
 103                         printk("kgm_add_recv: memory alloc failed\n");
 104                         return NULL;
 105         }
 106
 107         list_add(&conn->krx_item,(struct list_head *)&data->kgm_list);
 108         //        conn->ndx=ndx;
 109         //        conn->len=conn->ptlhdr_copied=0;
 110         //        conn->loopback=0;
 111         return conn;
 112 }
 113
 114 static nal_t *kgmnal_init(int interface, ptl_pt_index_t ptl_size,
 115                           ptl_ac_index_t  ac_size, ptl_pid_t requested_pid)
 116 {
 117         unsigned int nnids;
 118
 119         gm_max_node_id_in_use(kgmnal_data.kgm_port, &nnids);
 120
 121         CDEBUG(D_NET, "calling lib_init with nid 0x%Lx of %d\n",
 122                kgmnal_data.kgm_nid, nnids);
 123         lib_init(&kgmnal_lib, kgmnal_data.kgm_nid, 0, nnids,ptl_size, ac_size);
 124         return &kgmnal_api;
 125 }
 126
 127 static void /*__exit*/
 128 kgmnal_finalize(void)
 129 {
 130         struct list_head *tmp;
 131
 132         PORTAL_SYMBOL_UNREGISTER (kgmnal_ni);
 133         PtlNIFini(kgmnal_ni);
 134         lib_fini(&kgmnal_api);
 135
 136         if (kgmnal_data.kgm_port) {
 137                 gm_close(kgmnal_data.kgm_port);
 138         }
 139
 140         /* FIXME: free dma buffers */
 141         /* FIXME: kill receiver thread */
 142
 143         PORTAL_FREE (kgmnal_data.kgm_trans, bsizeof(kgmnal_tx_t)*TXMSGS);
 144
 145         list_for_each(tmp, &kgmnal_data.kgm_list) {
 146                 kgmnal_rx_t *conn;
 147                 conn = list_entry(tmp, kgmnal_rx_t, krx_item);
 148                 CDEBUG(D_IOCTL, "freeing conn %p\n",conn);
 149                 tmp = tmp->next;
 150                 list_del(&conn->krx_item);
 151                 PORTAL_FREE(conn, sizeof(*conn));
 152         }
 153
 154         CDEBUG (D_MALLOC, "done kmem %d\n", atomic_read (&portal_kmemory));
 155
 156         return;
 157 }
 158
 159 static int __init
 160 kgmnal_initialize(void)
 161 {
 162         int rc;
 163         int ntok;
 164         unsigned long sizemask;
 165         unsigned int nid;
 166
 167         CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read (&portal_kmemory));
 168
 169         kgmnal_api.forward = kgmnal_forward;
 170         kgmnal_api.shutdown = kgmnal_shutdown;
 171         kgmnal_api.yield = kgmnal_yield;
 172         kgmnal_api.validate = NULL;         /* our api validate is a NOOP */
 173         kgmnal_api.lock= kgmnal_lock;
 174         kgmnal_api.unlock= kgmnal_unlock;
 175         kgmnal_api.nal_data = &kgmnal_data;
 176
 177         kgmnal_lib.nal_data = &kgmnal_data;
 178
 179         memset(&kgmnal_data, 0, sizeof(kgmnal_data));
 180
 181         INIT_LIST_HEAD(&kgmnal_data.kgm_list);
 182         kgmnal_data.kgm_cb = &kgmnal_lib;
 183
 184         /* Allocate transmit descriptors */
 185         PORTAL_ALLOC (kgmnal_data.kgm_trans, sizeof(kgmnal_tx_t)*TXMSGS);
 186         if (kgmnal_data.kgm_trans==NULL) {
 187                 printk("kgmnal: init: failed to allocate transmit "
 188                        "descriptors\n");
 189                 return -1;
 190         }
 191         memset(kgmnal_data.kgm_trans,-1,sizeof(kgmnal_tx_t)*(TXMSGS));
 192
 193         spin_lock_init(&kgmnal_data.kgm_dispatch_lock);
 194         spin_lock_init(&kgmnal_data.kgm_update_lock);
 195         spin_lock_init(&kgmnal_data.kgm_send_lock);
 196
 197         /* Do the receiver and xmtr allocation */
 198
 199         rc = gm_init();
 200         if (rc != GM_SUCCESS) {
 201                 CERROR("gm_init failed: %d\n", rc);
 202                 return -1;
 203         }
 204
 205         rc = gm_open(&kgmnal_data.kgm_port, 0 , KGM_PORT_NUM, KGM_HOSTNAME,
 206                      GM_API_VERSION_1_1);
 207         if (rc != GM_SUCCESS) {
 208                 gm_finalize();
 209                 kgmnal_data.kgm_port = NULL;
 210                 CERROR("gm_open failed: %d\n", rc);
 211                 return -1;
 212         }
 213         gm_get_node_id(kgmnal_data.kgm_port, &nid);
 214         kgmnal_data.kgm_nid = nid;
 215         /* Allocate 2 different sizes of buffers. For new, use half
 216            the tokens for each. */
 217         ntok = gm_num_receive_tokens(kgmnal_data.kgm_port)/2;
 218         CDEBUG(D_NET, "gmnal_init: creating %d large %d byte recv buffers\n",
 219                ntok, MSG_LEN_LARGE);
 220         while (ntok-- > 0) {
 221                 void * buffer = gm_dma_malloc(kgmnal_data.kgm_port,
 222                                               MSG_LEN_LARGE);
 223                 if (buffer == NULL) {
 224                         CERROR("gm_init failed: %d\n", rc);
 225                         return (-ENOMEM);
 226                 }
 227                 CDEBUG(D_NET, " add buffer: port %p buf %p len %d size %d "
 228                        "pri %d\n ", kgmnal_data.kgm_port, buffer,
 229                        MSG_LEN_LARGE, MSG_SIZE_LARGE, GM_LOW_PRIORITY);
 230
 231                 gm_provide_receive_buffer(kgmnal_data.kgm_port, buffer,
 232                                           MSG_SIZE_LARGE, GM_LOW_PRIORITY);
 233         }
 234
 235         ntok = gm_num_receive_tokens(kgmnal_data.kgm_port)/2;
 236         CDEBUG(D_NET, "gmnal_init: creating %d small %d byte recv buffers\n",
 237                ntok, MSG_LEN_SMALL);
 238         while (ntok-- > 0) {
 239                 void * buffer = gm_dma_malloc(kgmnal_data.kgm_port,
 240                                               MSG_LEN_SMALL);
 241                 if (buffer == NULL) {
 242                         CERROR("gm_init failed: %d\n", rc);
 243                         return (-ENOMEM);
 244                 }
 245                 CDEBUG(D_NET, " add buffer: port %p buf %p len %d size %d "
 246                        "pri %d\n ", kgmnal_data.kgm_port, buffer,
 247                        MSG_LEN_SMALL, MSG_SIZE_SMALL, GM_LOW_PRIORITY);
 248
 249                 gm_provide_receive_buffer(kgmnal_data.kgm_port, buffer,
 250                                           MSG_SIZE_SMALL, GM_LOW_PRIORITY);
 251
 252         }
 253         sizemask = (1 << MSG_SIZE_LARGE) | (1 << MSG_SIZE_SMALL);
 254         CDEBUG(D_NET, "gm_set_acceptable_sizes port %p pri %d mask 0x%x\n",
 255                         kgmnal_data.kgm_port, GM_LOW_PRIORITY, sizemask);
 256         gm_set_acceptable_sizes(kgmnal_data.kgm_port, GM_LOW_PRIORITY,
 257                                 sizemask);
 258         gm_set_acceptable_sizes(kgmnal_data.kgm_port, GM_HIGH_PRIORITY, 0);
 259
 260         /* Initialize Network Interface */
 261         rc = PtlNIInit(kgmnal_init, 32, 4, 0, &kgmnal_ni);
 262         if (rc) {
 263                 CERROR("PtlNIInit failed %d\n", rc);
 264                 return (-ENOMEM);
 265         }
 266
 267         /* Start receiver thread */
 268         kernel_thread(kgmnal_recv_thread, &kgmnal_data, 0);
 269
 270         PORTAL_SYMBOL_REGISTER(kgmnal_ni);
 271
 272         kgmnal_data.kgm_init = 1;
 273
 274         return 0;
 275 }
 276
 277 MODULE_AUTHOR("Robert Read <rread@datarithm.net>");
 278 MODULE_DESCRIPTION("Kernel Myrinet GM NAL v0.1");
 279 MODULE_LICENSE("GPL");
 280
 281 module_init (kgmnal_initialize);
 282 module_exit (kgmnal_finalize);
 283
 284 EXPORT_SYMBOL (kgmnal_ni);