lustre/portals/knals/gmnal/gmnal_cb.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22
  23 /*
  24  *      This file implements the nal cb functions
  25  */
  26
  27
  28 #include "gmnal.h"
  29
  30 int gmnal_cb_recv(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie,
  31                    unsigned int niov, struct iovec *iov, size_t mlen,
  32                    size_t rlen)
  33 {
  34         gmnal_srxd_t    *srxd = (gmnal_srxd_t*)private;
  35         int             status = PTL_OK;
  36
  37
  38         CDEBUG(D_TRACE, "gmnal_cb_recv nal_cb [%p], private[%p], cookie[%p],
  39                niov[%d], iov [%p], mlen["LPSZ"], rlen["LPSZ"]\n",
  40                nal_cb, private, cookie, niov, iov, mlen, rlen);
  41
  42         switch(srxd->type) {
  43         case(GMNAL_SMALL_MESSAGE):
  44                 CDEBUG(D_INFO, "gmnal_cb_recv got small message\n");
  45                 status = gmnal_small_rx(nal_cb, private, cookie, niov,
  46                                          iov, mlen, rlen);
  47         break;
  48         case(GMNAL_LARGE_MESSAGE_INIT):
  49                 CDEBUG(D_INFO, "gmnal_cb_recv got large message init\n");
  50                 status = gmnal_large_rx(nal_cb, private, cookie, niov,
  51                                          iov, mlen, rlen);
  52         }
  53
  54
  55         CDEBUG(D_INFO, "gmnal_cb_recv gmnal_return status [%d]\n", status);
  56         return(status);
  57 }
  58
  59 int gmnal_cb_recv_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie,
  60                          unsigned int kniov, ptl_kiov_t *kiov, size_t mlen,
  61                          size_t rlen)
  62 {
  63         gmnal_srxd_t    *srxd = (gmnal_srxd_t*)private;
  64         int             status = PTL_OK;
  65         struct iovec    *iovec = NULL, *iovec_dup = NULL;
  66         int             i = 0;
  67
  68
  69         CDEBUG(D_TRACE, "gmnal_cb_recv_pages nal_cb [%p],private[%p],
  70                cookie[%p], kniov[%d], kiov [%p], mlen["LPSZ"], rlen["LPSZ"]\n",
  71                nal_cb, private, cookie, kniov, kiov, mlen, rlen);
  72
  73         if (srxd->type == GMNAL_SMALL_MESSAGE) {
  74                 PORTAL_ALLOC(iovec, sizeof(struct iovec)*kniov);
  75                 if (!iovec) {
  76                         CDEBUG(D_ERROR, "Can't malloc\n");
  77                         return(GMNAL_STATUS_FAIL);
  78                 }
  79                 iovec_dup = iovec;
  80
  81                 /*
  82                  *      map each page and create an iovec for it
  83                  */
  84                 for (i=0; i<kniov; i++) {
  85                         CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", i, kiov);
  86                         CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n",
  87                                kiov->kiov_page, kiov->kiov_len,
  88                                kiov->kiov_offset);
  89                         iovec->iov_len = kiov->kiov_len;
  90                         CDEBUG(D_INFO, "Calling kmap[%p]", kiov->kiov_page);
  91
  92                         iovec->iov_base = kmap(kiov->kiov_page) +
  93                                                   kiov->kiov_offset;
  94
  95                         CDEBUG(D_INFO, "iov_base is [%p]\n", iovec->iov_base);
  96                         iovec++;
  97                         kiov++;
  98                 }
  99                 CDEBUG(D_INFO, "calling gmnal_small_rx\n");
 100                 status = gmnal_small_rx(nal_cb, private, cookie, kniov,
 101                                          iovec_dup, mlen, rlen);
 102                 PORTAL_FREE(iovec_dup, sizeof(struct iovec)*kniov);
 103         }
 104
 105
 106         CDEBUG(D_INFO, "gmnal_return status [%d]\n", status);
 107         return(status);
 108 }
 109
 110
 111 int gmnal_cb_send(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie,
 112                    ptl_hdr_t *hdr, int type, ptl_nid_t nid, ptl_pid_t pid,
 113                    unsigned int niov, struct iovec *iov, size_t len)
 114 {
 115
 116         gmnal_data_t    *nal_data;
 117
 118
 119         CDEBUG(D_TRACE, "gmnal_cb_send niov[%d] len["LPSZ"] nid["LPU64"]\n",
 120                niov, len, nid);
 121         nal_data = nal_cb->nal_data;
 122
 123         if (GMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) {
 124                 CDEBUG(D_INFO, "This is a small message send\n");
 125                 gmnal_small_tx(nal_cb, private, cookie, hdr, type, nid, pid,
 126                                 niov, iov, len);
 127         } else {
 128                 CDEBUG(D_ERROR, "Large message send it is not supported\n");
 129                 lib_finalize(nal_cb, private, cookie);
 130                 return(PTL_FAIL);
 131                 gmnal_large_tx(nal_cb, private, cookie, hdr, type, nid, pid,
 132                                 niov, iov, len);
 133         }
 134         return(PTL_OK);
 135 }
 136
 137 int gmnal_cb_send_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie,
 138                          ptl_hdr_t *hdr, int type, ptl_nid_t nid, ptl_pid_t pid,                         unsigned int kniov, ptl_kiov_t *kiov, size_t len)
 139 {
 140
 141         int     i = 0;
 142         gmnal_data_t    *nal_data;
 143         struct  iovec   *iovec = NULL, *iovec_dup = NULL;
 144
 145         CDEBUG(D_TRACE, "gmnal_cb_send_pages nid ["LPU64"] niov[%d] len["LPSZ"]\n", nid, kniov, len);
 146         nal_data = nal_cb->nal_data;
 147         PORTAL_ALLOC(iovec, kniov*sizeof(struct iovec));
 148         iovec_dup = iovec;
 149         if (GMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) {
 150                 CDEBUG(D_INFO, "This is a small message send\n");
 151
 152                 for (i=0; i<kniov; i++) {
 153                         CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", i, kiov);
 154                         CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n",
 155                                kiov->kiov_page, kiov->kiov_len,
 156                                kiov->kiov_offset);
 157
 158                         iovec->iov_base = kmap(kiov->kiov_page)
 159                                                 + kiov->kiov_offset;
 160
 161                         iovec->iov_len = kiov->kiov_len;
 162                         iovec++;
 163                         kiov++;
 164                 }
 165                 gmnal_small_tx(nal_cb, private, cookie, hdr, type, nid,
 166                                 pid, kniov, iovec_dup, len);
 167         } else {
 168                 CDEBUG(D_ERROR, "Large message send it is not supported yet\n");
 169                 return(PTL_FAIL);
 170                 for (i=0; i<kniov; i++) {
 171                         CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", i, kiov);
 172                         CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n",
 173                                kiov->kiov_page, kiov->kiov_len,
 174                                kiov->kiov_offset);
 175
 176                         iovec->iov_base = kmap(kiov->kiov_page)
 177                                                  + kiov->kiov_offset;
 178                         iovec->iov_len = kiov->kiov_len;
 179                         iovec++;
 180                         kiov++;
 181                 }
 182                 gmnal_large_tx(nal_cb, private, cookie, hdr, type, nid,
 183                                 pid, kniov, iovec, len);
 184         }
 185         PORTAL_FREE(iovec_dup, kniov*sizeof(struct iovec));
 186         return(PTL_OK);
 187 }
 188
 189 int gmnal_cb_read(nal_cb_t *nal_cb, void *private, void *dst,
 190                    user_ptr src, size_t len)
 191 {
 192         gm_bcopy(src, dst, len);
 193         return(PTL_OK);
 194 }
 195
 196 int gmnal_cb_write(nal_cb_t *nal_cb, void *private, user_ptr dst,
 197                     void *src, size_t len)
 198 {
 199         gm_bcopy(src, dst, len);
 200         return(PTL_OK);
 201 }
 202
 203 int gmnal_cb_callback(nal_cb_t *nal_cb, void *private, lib_eq_t *eq,
 204                        ptl_event_t *ev)
 205 {
 206
 207         if (eq->event_callback != NULL) {
 208                 CDEBUG(D_INFO, "found callback\n");
 209                 eq->event_callback(ev);
 210         }
 211
 212         return(PTL_OK);
 213 }
 214
 215 void *gmnal_cb_malloc(nal_cb_t *nal_cb, size_t len)
 216 {
 217         void *ptr = NULL;
 218         CDEBUG(D_TRACE, "gmnal_cb_malloc len["LPSZ"]\n", len);
 219         PORTAL_ALLOC(ptr, len);
 220         return(ptr);
 221 }
 222
 223 void gmnal_cb_free(nal_cb_t *nal_cb, void *buf, size_t len)
 224 {
 225         CDEBUG(D_TRACE, "gmnal_cb_free :: buf[%p] len["LPSZ"]\n", buf, len);
 226         PORTAL_FREE(buf, len);
 227         return;
 228 }
 229
 230 void gmnal_cb_unmap(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov,
 231                      void **addrkey)
 232 {
 233         return;
 234 }
 235
 236 int  gmnal_cb_map(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov,
 237                    void**addrkey)
 238 {
 239         return(PTL_OK);
 240 }
 241
 242 void gmnal_cb_printf(nal_cb_t *nal_cb, const char *fmt, ...)
 243 {
 244         CDEBUG(D_TRACE, "gmnal_cb_printf\n");
 245         printk(fmt);
 246         return;
 247 }
 248
 249 void gmnal_cb_cli(nal_cb_t *nal_cb, unsigned long *flags)
 250 {
 251         gmnal_data_t    *nal_data = (gmnal_data_t*)nal_cb->nal_data;
 252
 253         spin_lock_irqsave(&nal_data->cb_lock, *flags);
 254         return;
 255 }
 256
 257 void gmnal_cb_sti(nal_cb_t *nal_cb, unsigned long *flags)
 258 {
 259         gmnal_data_t    *nal_data = (gmnal_data_t*)nal_cb->nal_data;
 260
 261         spin_unlock_irqrestore(&nal_data->cb_lock, *flags);
 262         return;
 263 }
 264
 265 int gmnal_cb_dist(nal_cb_t *nal_cb, ptl_nid_t nid, unsigned long *dist)
 266 {
 267         CDEBUG(D_TRACE, "gmnal_cb_dist\n");
 268         if (dist)
 269                 *dist = 27;
 270         return(PTL_OK);
 271 }