1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2003 Los Alamos National Laboratory (LANL)
6 * This file is part of Lustre, http://www.lustre.org/
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 * This file implements the nal cb functions
28 int lgmnal_cb_recv(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, unsigned int niov, struct iovec *iov, size_t mlen, size_t rlen)
30 lgmnal_srxd_t *srxd = (lgmnal_srxd_t*)private;
32 lgmnal_data_t *nal_data = nal_cb->nal_data;
35 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_recv nal_cb [%p],private[%p], cookie[%p], niov[%d], iov [%p], mlen[%d], rlen[%d]\n", nal_cb, private, cookie, niov, iov, mlen, rlen));
37 if (srxd->type == LGMNAL_SMALL_MESSAGE) {
38 if (!LGMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, mlen)) {
39 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("lgmnal_cb_recv. This is not a small message\n"));
41 status = lgmnal_small_receive2(nal_cb, private, cookie, niov, iov, mlen, rlen);
45 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("lgmnal_return status [%d]\n", status));
49 int lgmnal_cb_recv_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, unsigned int kniov, ptl_kiov_t *kiov, size_t mlen, size_t rlen)
51 lgmnal_srxd_t *srxd = (lgmnal_srxd_t*)private;
53 struct iovec *iovec = NULL;
57 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_recv_pages nal_cb [%p],private[%p], cookie[%p], kniov[%d], kiov [%p], mlen[%d], rlen[%d]\n", nal_cb, private, cookie, kniov, kiov, mlen, rlen));
59 if (srxd->type == LGMNAL_SMALL_MESSAGE) {
60 PORTAL_ALLOC(iovec, sizeof(struct iovec)*kniov);
62 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Can't malloc\n"));
63 return(LGMNAL_STATUS_FAIL);
67 * map each page and create an iovec for it
69 for (i=0; i<kniov; i++) {
70 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("processing kniov [%d] [%p]\n", i, kiov));
71 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("kniov page [%p] len [%d] offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset));
72 iovec->iov_len = kiov->kiov_len;
73 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Calling kmap", kiov->kiov_page));
74 iovec->iov_base = kmap(kiov->kiov_page) + kiov->kiov_offset;
75 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Calling iov_base is [%p]", iovec->iov_base));
76 iovec->iov_len = kiov->kiov_len;
78 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("calling lgmnal_small_receive2\n"));
79 status = lgmnal_small_receive2(nal_cb, private, cookie, kniov, iovec, mlen, rlen);
80 PORTAL_FREE(iovec, sizeof(struct iovec)*kniov);
84 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("lgmnal_return status [%d]\n", status));
89 int lgmnal_cb_send(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, ptl_hdr_t *hdr,
90 int type, ptl_nid_t nid, ptl_pid_t pid, unsigned int niov, struct iovec *iov, size_t len)
93 lgmnal_data_t *nal_data;
96 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_sendnid [%lu] niov[%d] len[%d]\n", nid, niov, len));
97 nal_data = nal_cb->nal_data;
99 if (LGMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) {
100 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("This is a small message send\n"));
101 lgmnal_small_transmit(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
103 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("This is a large message send it is not supported yet\n"));
105 lgmnal_large_transmit1(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
107 return(LGMNAL_STATUS_FAIL);
112 int lgmnal_cb_send_pages(nal_cb_t *nal_cb, void *private, lib_msg_t *cookie, ptl_hdr_t *hdr,
113 int type, ptl_nid_t nid, ptl_pid_t pid, unsigned int kniov, ptl_kiov_t *kiov, size_t len)
117 lgmnal_data_t *nal_data;
120 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_send_pages nid [%lu] niov[%d] len[%d]\n", nid, kniov, len));
121 nal_data = nal_cb->nal_data;
122 if (LGMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) {
123 /* TO DO fix small message for send pages */
124 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("This is a small message send\n"));
125 PORTAL_ALLOC(iovec, kniov*sizeof(struct iovec));
127 for (i=0; i<kniov; i++) {
128 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("processing kniov [%d] [%p]\n", i, kiov));
129 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("kniov page [%p] len [%d] offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset));
130 iovec->iov_len = kiov->kiov_len;
131 iovec->iov_base = kmap(kiov->kiov_page) + kiov->kiov_offset;
132 iovec->iov_len = kiov->kiov_len;
134 lgmnal_small_transmit(nal_cb, private, cookie, hdr, type, nid, pid, kniov, iovec, len);
135 PORTAL_FREE(iovec, kniov*sizeof(struct iovec));
137 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("This is a large message send it is not supported yet\n"));
139 lgmnal_large_transmit1(nal_cb, private, cookie, hdr, type, nid, pid, niov, iov, len);
141 return(LGMNAL_STATUS_FAIL);
146 int lgmnal_cb_read(nal_cb_t *nal_cb, void *private, void *dst, user_ptr src, size_t len)
148 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_read dst [%p] src [%p] len[%d]\n", dst, src, len));
149 gm_bcopy(src, dst, len);
153 int lgmnal_cb_write(nal_cb_t *nal_cb, void *private, user_ptr dst, void *src, size_t len)
155 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_write :: dst [%p] src [%p] len[%d]\n", dst, src, len));
156 gm_bcopy(src, dst, len);
160 int lgmnal_cb_callback(nal_cb_t *nal_cb, void *private, lib_eq_t *eq, ptl_event_t *ev)
162 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_callback nal_cb[%p], private[%p], eq[%p], ev[%p]\n", nal_cb, private, eq, ev));
164 if (eq->event_callback != NULL) {
165 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("found callback\n"));
166 eq->event_callback(ev);
172 void *lgmnal_cb_malloc(nal_cb_t *nal_cb, size_t len)
175 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_malloc len[%d]\n", len));
176 PORTAL_ALLOC(ptr, len);
180 void lgmnal_cb_free(nal_cb_t *nal_cb, void *buf, size_t len)
182 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_free :: buf[%p] len[%d]\n", buf, len));
183 PORTAL_FREE(buf, len);
187 void lgmnal_cb_unmap(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov, void **addrkey)
189 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_unmap niov[%d] iov[%], addrkey[%p]\n", niov, iov, addrkey));
193 int lgmnal_cb_map(nal_cb_t *nal_cb, unsigned int niov, struct iovec *iov, void**addrkey)
195 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_map niov[%d], iov[%p], addrkey[%p], niov, iov, addrkey\n"));
199 void lgmnal_cb_printf(nal_cb_t *nal_cb, const char *fmt, ...)
201 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_printf\n"));
206 void lgmnal_cb_cli(nal_cb_t *nal_cb, unsigned long *flags)
208 lgmnal_data_t *nal_data = (lgmnal_data_t*)nal_cb->nal_data;
209 spinlock_t cb_lock = nal_data->cb_lock;
210 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_cli\n"));
212 local_irq_save(*flags);
213 spin_lock_irqsave(&cb_lock, *flags);
219 void lgmnal_cb_sti(nal_cb_t *nal_cb, unsigned long *flags)
221 lgmnal_data_t *nal_data = (lgmnal_data_t*)nal_cb->nal_data;
222 spinlock_t cb_lock = nal_data->cb_lock;
225 local_irq_restore(*flags);
226 spin_unlock_irqrestore(&cb_lock, *flags);
228 spin_unlock(&cb_lock);
229 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_sti\n"));
233 int lgmnal_cb_dist(nal_cb_t *nal_cb, ptl_nid_t nid, unsigned long *dist)
235 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_cb_dist\n"));
244 EXPORT_SYMBOL(lgmnal_cb_send);
245 EXPORT_SYMBOL(lgmnal_cb_send_pages);
246 EXPORT_SYMBOL(lgmnal_cb_recv);
247 EXPORT_SYMBOL(lgmnal_cb_recv_pages);
248 EXPORT_SYMBOL(lgmnal_cb_read);
249 EXPORT_SYMBOL(lgmnal_cb_write);
250 EXPORT_SYMBOL(lgmnal_cb_cli);
251 EXPORT_SYMBOL(lgmnal_cb_sti);
252 EXPORT_SYMBOL(lgmnal_cb_dist);
253 EXPORT_SYMBOL(lgmnal_cb_printf);
254 EXPORT_SYMBOL(lgmnal_cb_map);
255 EXPORT_SYMBOL(lgmnal_cb_unmap);
256 EXPORT_SYMBOL(lgmnal_cb_callback);
257 EXPORT_SYMBOL(lgmnal_cb_free);
258 EXPORT_SYMBOL(lgmnal_cb_malloc);