lnet/klnds/lgmlnd/lgmnal_cb.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21 /*
  22  *      This file implements the nal cb functions
  23  */
  24
  25
  26 #include "lgmnal.h"
  27
  28 int lgmnal_cb_recv(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, unsigned int niov, struct iovec *iov, size_t mlen, size_t rlen)
  29 {
  30         lgmnal_srxd_t   *srxd = (lgmnal_srxd_t*)private;
  31         int             status = PTL_OK;
  32         lgmnal_data_t   *nal_data = nal_cb->nal_data;
  33
  34
  35         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_recv nal_cb [%p],private[%p], cookie[%p], niov[%d], iov [%p], mlen[%d], rlen[%d]\n", nal_cb, private, cookie, niov, iov, mlen, rlen));
  36
  37         if (srxd->type == LGMNAL_SMALL_MESSAGE) {
  38                 if (!LGMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, mlen)) {
  39                         LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("lgmnal_cb_recv. This is not a small message\n"));
  40                 }
  41                 status = lgmnal_small_receive2(nal_cb, private, cookie, niov, iov, mlen, rlen);
  42         }
  43
  44
  45         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("lgmnal_return status [%d]\n", status));
  46         return(status);
  47 }
  48
  49 int lgmnal_cb_recv_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, unsigned int kniov, ptl_kiov_t *kiov, size_t mlen, size_t rlen)
  50 {
  51         lgmnal_srxd_t   *srxd = (lgmnal_srxd_t*)private;
  52         int             status = PTL_OK;
  53         struct iovec    *iovec = NULL;
  54         int             i = 0;
  55
  56
  57         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_recv_pages nal_cb [%p],private[%p], cookie[%p], kniov[%d], kiov [%p], mlen[%d], rlen[%d]\n", nal_cb, private, cookie, kniov, kiov, mlen, rlen));
  58
  59         if (srxd->type == LGMNAL_SMALL_MESSAGE) {
  60                 PORTAL_ALLOC(iovec, sizeof(struct iovec)*kniov);
  61                 if (!iovec) {
  62                         LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Can't malloc\n"));
  63                         return(LGMNAL_STATUS_FAIL);
  64                 }
  65
  66                 /*
  67                  *      map each page and create an iovec for it
  68                  */
  69                 for (i=0; i<kniov; i++) {
  70                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("processing kniov [%d] [%p]\n", i, kiov));
  71                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("kniov page [%p] len [%d] offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset));
  72                         iovec->iov_len = kiov->kiov_len;
  73                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Calling kmap", kiov->kiov_page));
  74                         iovec->iov_base = kmap(kiov->kiov_page) + kiov->kiov_offset;
  75                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Calling iov_base is [%p]", iovec->iov_base));
  76                         iovec->iov_len = kiov->kiov_len;
  77                 }
  78                 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("calling lgmnal_small_receive2\n"));
  79                 status = lgmnal_small_receive2(nal_cb, private, cookie, kniov, iovec, mlen, rlen);
  80                 PORTAL_FREE(iovec, sizeof(struct iovec)*kniov);
  81         }
  82
  83
  84         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("lgmnal_return status [%d]\n", status));
  85         return(status);
  86 }
  87
  88
  89 int lgmnal_cb_send(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, ptl_hdr_t *hdr,
  90         int type, ptl_nid_t nid, ptl_pid_t pid, unsigned int niov, struct iovec *iov, size_t len)
  91 {
  92
  93         lgmnal_data_t   *nal_data;
  94
  95
  96         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_sendnid [%lu] niov[%d] len[%d]\n", nid, niov, len));
  97         nal_data = nal_cb->nal_data;
  98
  99         if (LGMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) {
 100                 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("This is a small message send\n"));
 101                 lgmnal_small_transmit(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
 102         } else {
 103                 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("This is a large message send it is not supported yet\n"));
 104 /*
 105                 lgmnal_large_transmit1(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
 106 */
 107                 return(LGMNAL_STATUS_FAIL);
 108         }
 109         return(PTL_OK);
 110 }
 111
 112 int lgmnal_cb_send_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, ptl_hdr_t *hdr,
 113         int type, ptl_nid_t nid, ptl_pid_t pid, unsigned int kniov, ptl_kiov_t *kiov, size_t len)
 114 {
 115
 116         int     i = 0;
 117         lgmnal_data_t   *nal_data;
 118         struct  iovec   *iovec;
 119
 120         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_send_pages nid [%lu] niov[%d] len[%d]\n", nid, kniov, len));
 121         nal_data = nal_cb->nal_data;
 122         if (LGMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) {
 123                 /* TO DO fix small message for send pages */
 124                 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("This is a small message send\n"));
 125                 PORTAL_ALLOC(iovec, kniov*sizeof(struct iovec));
 126
 127                 for (i=0; i<kniov; i++) {
 128                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("processing kniov [%d] [%p]\n", i, kiov));
 129                         LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("kniov page [%p] len [%d] offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset));
 130                         iovec->iov_len = kiov->kiov_len;
 131                         iovec->iov_base = kmap(kiov->kiov_page) + kiov->kiov_offset;
 132                         iovec->iov_len = kiov->kiov_len;
 133                 }
 134                 lgmnal_small_transmit(nal_cb, private, cookie, hdr, type, nid, pid, kniov, iovec, len);
 135                 PORTAL_FREE(iovec, kniov*sizeof(struct iovec));
 136         } else {
 137                 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("This is a large message send it is not supported yet\n"));
 138 /*
 139                 lgmnal_large_transmit1(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
 140 */
 141                 return(LGMNAL_STATUS_FAIL);
 142         }
 143         return(PTL_OK);
 144 }
 145
 146 int lgmnal_cb_read(nal_cb_t *nal_cb, void *private, void *dst, user_ptr src, size_t len)
 147 {
 148         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_read dst [%p] src [%p] len[%d]\n", dst, src, len));
 149         gm_bcopy(src, dst, len);
 150         return(PTL_OK);
 151 }
 152
 153 int lgmnal_cb_write(nal_cb_t *nal_cb, void *private, user_ptr dst, void *src, size_t len)
 154 {
 155         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_write :: dst [%p] src [%p] len[%d]\n", dst, src, len));
 156         gm_bcopy(src, dst, len);
 157         return(PTL_OK);
 158 }
 159
 160 int lgmnal_cb_callback(nal_cb_t *nal_cb, void *private, lib_eq_t *eq, ptl_event_t *ev)
 161 {
 162         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_callback nal_cb[%p], private[%p], eq[%p], ev[%p]\n", nal_cb, private, eq, ev));
 163
 164         if (eq->event_callback != NULL) {
 165                 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("found callback\n"));
 166                 eq->event_callback(ev);
 167         }
 168
 169         return(PTL_OK);
 170 }
 171
 172 void *lgmnal_cb_malloc(nal_cb_t *nal_cb, size_t len)
 173 {
 174         void *ptr = NULL;
 175         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_malloc len[%d]\n", len));
 176         PORTAL_ALLOC(ptr, len);
 177         return(ptr);
 178 }
 179
 180 void lgmnal_cb_free(nal_cb_t *nal_cb, void *buf, size_t len)
 181 {
 182         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_free :: buf[%p] len[%d]\n", buf, len));
 183         PORTAL_FREE(buf, len);
 184         return;
 185 }
 186
 187 void lgmnal_cb_unmap(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov, void **addrkey)
 188 {
 189         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_unmap niov[%d] iov[%], addrkey[%p]\n", niov, iov, addrkey));
 190         return;
 191 }
 192
 193 int  lgmnal_cb_map(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov, void**addrkey)
 194 {
 195         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_map niov[%d], iov[%p], addrkey[%p], niov, iov, addrkey\n"));
 196         return(PTL_OK);
 197 }
 198
 199 void lgmnal_cb_printf(nal_cb_t *nal_cb, const char *fmt, ...)
 200 {
 201         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_printf\n"));
 202         lgmnal_print(fmt);
 203         return;
 204 }
 205
 206 void lgmnal_cb_cli(nal_cb_t *nal_cb, unsigned long *flags)
 207 {
 208         lgmnal_data_t   *nal_data = (lgmnal_data_t*)nal_cb->nal_data;
 209         spinlock_t      cb_lock = nal_data->cb_lock;
 210         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_cli\n"));
 211 /*
 212         local_irq_save(*flags);
 213         spin_lock_irqsave(&cb_lock, *flags);
 214 */
 215         spin_lock(&cb_lock);
 216         return;
 217 }
 218
 219 void lgmnal_cb_sti(nal_cb_t *nal_cb, unsigned long *flags)
 220 {
 221         lgmnal_data_t   *nal_data = (lgmnal_data_t*)nal_cb->nal_data;
 222         spinlock_t      cb_lock = nal_data->cb_lock;
 223
 224 /*
 225         local_irq_restore(*flags);
 226         spin_unlock_irqrestore(&cb_lock, *flags);
 227 */
 228         spin_unlock(&cb_lock);
 229         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_sti\n"));
 230         return;
 231 }
 232
 233 int lgmnal_cb_dist(nal_cb_t *nal_cb, ptl_nid_t nid, unsigned long *dist)
 234 {
 235         LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_dist\n"));
 236         if (dist)
 237                 *dist = 27;
 238         return(PTL_OK);
 239 }
 240
 241
 242
 243
 244 EXPORT_SYMBOL(lgmnal_cb_send);
 245 EXPORT_SYMBOL(lgmnal_cb_send_pages);
 246 EXPORT_SYMBOL(lgmnal_cb_recv);
 247 EXPORT_SYMBOL(lgmnal_cb_recv_pages);
 248 EXPORT_SYMBOL(lgmnal_cb_read);
 249 EXPORT_SYMBOL(lgmnal_cb_write);
 250 EXPORT_SYMBOL(lgmnal_cb_cli);
 251 EXPORT_SYMBOL(lgmnal_cb_sti);
 252 EXPORT_SYMBOL(lgmnal_cb_dist);
 253 EXPORT_SYMBOL(lgmnal_cb_printf);
 254 EXPORT_SYMBOL(lgmnal_cb_map);
 255 EXPORT_SYMBOL(lgmnal_cb_unmap);
 256 EXPORT_SYMBOL(lgmnal_cb_callback);
 257 EXPORT_SYMBOL(lgmnal_cb_free);
 258 EXPORT_SYMBOL(lgmnal_cb_malloc);