#include <asm/page.h>
#include <linux/nmi.h>
#include <linux/pagemap.h>
+
+#include <libcfs/linux/linux-mem.h>
+
#include "gnilnd.h"
/* this is useful when needed to debug wire corruption. */
}
}
-void kgnilnd_schedule_device_timer(unsigned long arg)
+void kgnilnd_schedule_device_timer(cfs_timer_cb_arg_t data)
{
- kgn_device_t *dev = (kgn_device_t *) arg;
+ kgn_device_t *dev = cfs_from_timer(dev, data, gnd_map_timer);
+
+ kgnilnd_schedule_device(dev);
+}
+
+void kgnilnd_schedule_device_timer_rd(cfs_timer_cb_arg_t data)
+{
+ kgn_device_t *dev = cfs_from_timer(dev, data, gnd_rdmaq_timer);
kgnilnd_schedule_device(dev);
}
if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ALLOC_TX))
return tx;
- tx = kmem_cache_alloc(kgnilnd_data.kgn_tx_cache, GFP_ATOMIC);
+ tx = kmem_cache_zalloc(kgnilnd_data.kgn_tx_cache, GFP_ATOMIC);
if (tx == NULL) {
CERROR("failed to allocate tx\n");
return NULL;
CDEBUG(D_MALLOC, "slab-alloced 'tx': %lu at %p.\n",
sizeof(*tx), tx);
- /* need this memset, cache alloc'd memory is not cleared */
- memset(tx, 0, sizeof(*tx));
-
/* setup everything here to minimize time under the lock */
tx->tx_buftype = GNILND_BUF_NONE;
tx->tx_msg.gnm_type = GNILND_MSG_NONE;
#define _kgnilnd_cksum(seed, ptr, nob) csum_partial(ptr, nob, seed)
/* we don't use offset as every one is passing a buffer reference that already
- * includes the offset into the base address -
- * see kgnilnd_setup_virt_buffer and kgnilnd_setup_immediate_buffer */
+ * includes the offset into the base address.
+ */
static inline __u16
kgnilnd_cksum(void *ptr, size_t nob)
{
kgnilnd_queue_tx(conn, tx);
}
-int
+static int
kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov,
- struct kvec *iov, struct bio_vec *kiov,
+ struct bio_vec *kiov,
unsigned int offset, unsigned int nob)
{
kgn_msg_t *msg = &tx->tx_msg;
if (nob == 0) {
tx->tx_buffer = NULL;
- } else if (kiov != NULL) {
+ } else {
- if ((niov > 0) && unlikely(niov > (nob/PAGE_SIZE))) {
- niov = round_up(nob + offset + kiov->bv_offset,
- PAGE_SIZE);
- }
+ if (niov && niov > (nob >> PAGE_SHIFT))
+ niov = DIV_ROUND_UP(nob + offset + kiov->bv_offset,
+ PAGE_SIZE);
LASSERTF(niov > 0 && niov < GNILND_MAX_IMMEDIATE/PAGE_SIZE,
- "bad niov %d msg %p kiov %p iov %p offset %d nob%d\n",
- niov, msg, kiov, iov, offset, nob);
+ "bad niov %d msg %p kiov %p offset %d nob%d\n",
+ niov, msg, kiov, offset, nob);
while (offset >= kiov->bv_len) {
offset -= kiov->bv_len;
tx->tx_buftype = GNILND_BUF_IMMEDIATE_KIOV;
tx->tx_nob = nob;
- } else {
- /* For now this is almost identical to kgnilnd_setup_virt_buffer, but we
- * could "flatten" the payload into a single contiguous buffer ready
- * for sending direct over an FMA if we ever needed to. */
-
- LASSERT(niov > 0);
-
- while (offset >= iov->iov_len) {
- offset -= iov->iov_len;
- niov--;
- iov++;
- LASSERT(niov > 0);
- }
-
- if (nob > iov->iov_len - offset) {
- CERROR("Can't handle multiple vaddr fragments\n");
- return -EMSGSIZE;
- }
-
- tx->tx_buffer = (void *)(((unsigned long)iov->iov_base) + offset);
-
- tx->tx_buftype = GNILND_BUF_IMMEDIATE;
- tx->tx_nob = nob;
}
/* checksum payload early - it shouldn't be changing after lnd_send */
}
int
-kgnilnd_setup_virt_buffer(kgn_tx_t *tx,
- unsigned int niov, struct kvec *iov,
- unsigned int offset, unsigned int nob)
-
-{
- LASSERT(nob > 0);
- LASSERT(niov > 0);
- LASSERT(tx->tx_buftype == GNILND_BUF_NONE);
-
- while (offset >= iov->iov_len) {
- offset -= iov->iov_len;
- niov--;
- iov++;
- LASSERT(niov > 0);
- }
-
- if (nob > iov->iov_len - offset) {
- CERROR("Can't handle multiple vaddr fragments\n");
- return -EMSGSIZE;
- }
-
- tx->tx_buftype = GNILND_BUF_VIRT_UNMAPPED;
- tx->tx_nob = nob;
- tx->tx_buffer = (void *)(((unsigned long)iov->iov_base) + offset);
- return 0;
-}
-
-int
kgnilnd_setup_phys_buffer(kgn_tx_t *tx, int nkiov, struct bio_vec *kiov,
unsigned int offset, unsigned int nob)
{
static inline int
kgnilnd_setup_rdma_buffer(kgn_tx_t *tx, unsigned int niov,
- struct kvec *iov, struct bio_vec *kiov,
+ struct bio_vec *kiov,
unsigned int offset, unsigned int nob)
{
- int rc;
-
- LASSERTF((iov == NULL) != (kiov == NULL), "iov 0x%p, kiov 0x%p, tx 0x%p,"
- " offset %d, nob %d, niov %d\n"
- , iov, kiov, tx, offset, nob, niov);
-
- if (kiov != NULL) {
- rc = kgnilnd_setup_phys_buffer(tx, niov, kiov, offset, nob);
- } else {
- rc = kgnilnd_setup_virt_buffer(tx, niov, iov, offset, nob);
- }
- return rc;
+ return kgnilnd_setup_phys_buffer(tx, niov, kiov, offset, nob);
}
/* kgnilnd_parse_lnet_rdma()
kgnilnd_dump_blob(D_BUFFS, "RDMA payload",
tx->tx_buffer, nob);
}
- /* fall through to dump log */
+ /* fallthrough */
case 1:
libcfs_debug_dumplog();
break;
dev->gnd_map_nphys++;
dev->gnd_map_physnop += tx->tx_phys_npages;
break;
-
- case GNILND_BUF_VIRT_MAPPED:
- bytes = tx->tx_nob;
- dev->gnd_map_nvirt++;
- dev->gnd_map_virtnob += tx->tx_nob;
- break;
}
if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK ||
dev->gnd_map_nphys--;
dev->gnd_map_physnop -= tx->tx_phys_npages;
break;
-
- case GNILND_BUF_VIRT_UNMAPPED:
- bytes = tx->tx_nob;
- dev->gnd_map_nvirt--;
- dev->gnd_map_virtnob -= tx->tx_nob;
- break;
}
if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK ||
tx->tx_msg.gnm_type == GNILND_MSG_GET_REQ) {
atomic64_sub(bytes, &dev->gnd_rdmaq_bytes_out);
LASSERTF(atomic64_read(&dev->gnd_rdmaq_bytes_out) >= 0,
- "bytes_out negative! %ld\n", atomic64_read(&dev->gnd_rdmaq_bytes_out));
+ "bytes_out negative! %lld\n",
+ (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out));
GNIDBG_TX(D_NETTRACE, tx, "rdma -- %d to %lld",
- bytes, atomic64_read(&dev->gnd_rdmaq_bytes_out));
+ bytes, (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out));
}
atomic_dec(&dev->gnd_n_mdd);
case GNILND_BUF_IMMEDIATE:
case GNILND_BUF_IMMEDIATE_KIOV:
case GNILND_BUF_PHYS_MAPPED:
- case GNILND_BUF_VIRT_MAPPED:
return 0;
case GNILND_BUF_PHYS_UNMAPPED:
* - this needs to turn into a non-fatal error soon to allow
* GART resource, etc starvation handling */
if (rrc != GNI_RC_SUCCESS) {
- GNIDBG_TX(D_NET, tx, "Can't map %d pages: dev %d "
- "phys %u pp %u, virt %u nob %llu",
+ GNIDBG_TX(D_NET, tx,
+ "Can't map %d pages: dev %d phys %u pp %u",
tx->tx_phys_npages, dev->gnd_id,
- dev->gnd_map_nphys, dev->gnd_map_physnop,
- dev->gnd_map_nvirt, dev->gnd_map_virtnob);
+ dev->gnd_map_nphys, dev->gnd_map_physnop);
RETURN(rrc == GNI_RC_ERROR_RESOURCE ? -ENOMEM : -EINVAL);
}
tx->tx_buftype = GNILND_BUF_PHYS_MAPPED;
kgnilnd_mem_add_map_list(dev, tx);
return 0;
-
- case GNILND_BUF_VIRT_UNMAPPED:
- rrc = kgnilnd_mem_register(dev->gnd_handle,
- (__u64)tx->tx_buffer, tx->tx_nob,
- NULL, flags, &tx->tx_map_key);
- if (rrc != GNI_RC_SUCCESS) {
- GNIDBG_TX(D_NET, tx, "Can't map %u bytes: dev %d "
- "phys %u pp %u, virt %u nob %llu",
- tx->tx_nob, dev->gnd_id,
- dev->gnd_map_nphys, dev->gnd_map_physnop,
- dev->gnd_map_nvirt, dev->gnd_map_virtnob);
- RETURN(rrc == GNI_RC_ERROR_RESOURCE ? -ENOMEM : -EINVAL);
- }
-
- tx->tx_buftype = GNILND_BUF_VIRT_MAPPED;
- kgnilnd_mem_add_map_list(dev, tx);
- if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK ||
- tx->tx_msg.gnm_type == GNILND_MSG_GET_REQ) {
- atomic64_add(tx->tx_nob, &dev->gnd_rdmaq_bytes_out);
- GNIDBG_TX(D_NETTRACE, tx, "rdma ++ %d to %ld\n",
- tx->tx_nob, atomic64_read(&dev->gnd_rdmaq_bytes_out));
- }
-
- return 0;
}
}
/* code below relies on +1 relationship ... */
BUILD_BUG_ON(GNILND_BUF_PHYS_MAPPED !=
(GNILND_BUF_PHYS_UNMAPPED + 1));
- BUILD_BUG_ON(GNILND_BUF_VIRT_MAPPED !=
- (GNILND_BUF_VIRT_UNMAPPED + 1));
switch (tx->tx_buftype) {
default:
case GNILND_BUF_NONE:
case GNILND_BUF_IMMEDIATE:
case GNILND_BUF_PHYS_UNMAPPED:
- case GNILND_BUF_VIRT_UNMAPPED:
break;
case GNILND_BUF_IMMEDIATE_KIOV:
if (tx->tx_phys != NULL) {
break;
case GNILND_BUF_PHYS_MAPPED:
- case GNILND_BUF_VIRT_MAPPED:
LASSERT(tx->tx_conn != NULL);
dev = tx->tx_conn->gnc_device;
/* it was sent, break out of switch to avoid default case of queueing */
break;
}
- /* needs to queue to try again, so fall through to default case */
+ /* needs to queue to try again, so... */
+ /* fall through... */
case GNILND_MSG_NOOP:
/* Just make sure this goes out first for this conn */
add_tail = 0;
kgn_net_t *net = ni->ni_data;
kgn_tx_t *tx;
int rc = 0;
- int mpflag = 0;
+ /* '1' for consistency with code that checks !mpflag to restore */
+ unsigned int mpflag = 1;
int reverse_rdma_flag = *kgnilnd_tunables.kgn_reverse_rdma;
/* NB 'private' is different depending on what we're sending.... */
"lntmsg %p niov %d\n", lntmsg, niov);
if (msg_vmflush)
- mpflag = cfs_memory_pressure_get_and_set();
+ mpflag = memalloc_noreclaim_save();
switch (type) {
default:
goto out;
}
rc = kgnilnd_setup_rdma_buffer(tx, lntmsg->msg_md->md_niov,
- NULL, lntmsg->msg_md->md_kiov,
+ lntmsg->msg_md->md_kiov,
0, lntmsg->msg_md->md_length);
if (rc != 0) {
CERROR("unable to setup buffer: %d\n", rc);
goto out;
}
- rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL,
+ rc = kgnilnd_setup_rdma_buffer(tx, niov,
kiov, offset, nob);
if (rc != 0) {
kgnilnd_tx_done(tx, rc);
goto out;
}
- rc = kgnilnd_setup_immediate_buffer(tx, niov, NULL, kiov, offset, nob);
+ rc = kgnilnd_setup_immediate_buffer(tx, niov, kiov, offset, nob);
if (rc != 0) {
kgnilnd_tx_done(tx, rc);
goto out;
out:
/* use stored value as we could have already finalized lntmsg here from a failed launch */
if (msg_vmflush)
- cfs_memory_pressure_restore(mpflag);
+ memalloc_noreclaim_restore(mpflag);
return rc;
}
if (rc != 0)
goto failed_1;
- rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL, kiov, offset, nob);
+ rc = kgnilnd_setup_rdma_buffer(tx, niov, kiov, offset, nob);
if (rc != 0)
goto failed_1;
int
kgnilnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg,
int delayed, unsigned int niov,
- struct kvec *iov, struct bio_vec *kiov,
+ struct bio_vec *kiov,
unsigned int offset, unsigned int mlen, unsigned int rlen)
{
kgn_rx_t *rx = private;
LASSERT(!in_interrupt());
LASSERTF(mlen <= rlen, "%d <= %d\n", mlen, rlen);
- /* Either all pages or all vaddrs */
- LASSERTF(!(kiov != NULL && iov != NULL), "kiov %p iov %p\n",
- kiov, iov);
GNIDBG_MSG(D_NET, rxmsg, "conn %p, rxmsg %p, lntmsg %p"
- " niov=%d kiov=%p iov=%p offset=%d mlen=%d rlen=%d",
+ " niov=%d kiov=%p offset=%d mlen=%d rlen=%d",
conn, rxmsg, lntmsg,
- niov, kiov, iov, offset, mlen, rlen);
+ niov, kiov, offset, mlen, rlen);
/* we need to lock here as recv can be called from any context */
read_lock(&kgnilnd_data.kgn_peer_conn_lock);
switch (rxmsg->gnm_type) {
default:
GNIDBG_MSG(D_NETERROR, rxmsg, "conn %p, rx %p, rxmsg %p, lntmsg %p"
- " niov=%d kiov=%p iov=%p offset=%d mlen=%d rlen=%d",
- conn, rx, rxmsg, lntmsg, niov, kiov, iov, offset, mlen, rlen);
+ " niov=%d kiov=%p offset=%d mlen=%d rlen=%d",
+ conn, rx, rxmsg, lntmsg, niov, kiov, offset, mlen, rlen);
LBUG();
case GNILND_MSG_IMMEDIATE:
case 2:
kgnilnd_dump_blob(D_BUFFS, "bad payload checksum",
&rxmsg[1], rxmsg->gnm_payload_len);
- /* fall through to dump */
+ /* fallthrough */
case 1:
libcfs_debug_dumplog();
break;
}
}
- if (kiov != NULL)
- lnet_copy_flat2kiov(
- niov, kiov, offset,
- *kgnilnd_tunables.kgn_max_immediate,
- &rxmsg[1], 0, mlen);
- else
- lnet_copy_flat2iov(
- niov, iov, offset,
- *kgnilnd_tunables.kgn_max_immediate,
- &rxmsg[1], 0, mlen);
+ lnet_copy_flat2kiov(
+ niov, kiov, offset,
+ *kgnilnd_tunables.kgn_max_immediate,
+ &rxmsg[1], 0, mlen);
kgnilnd_consume_rx(rx);
lnet_finalize(lntmsg, 0);
GOTO(nak_put_req, rc);
}
- rc = kgnilnd_setup_rdma_buffer(tx, niov, iov, kiov, offset, mlen);
+ rc = kgnilnd_setup_rdma_buffer(tx, niov,
+ kiov, offset, mlen);
if (rc != 0) {
GOTO(nak_put_req, rc);
}
if (rc != 0)
GOTO(nak_get_req_rev, rc);
-
- rc = kgnilnd_setup_rdma_buffer(tx, niov, iov, kiov, offset, mlen);
+ rc = kgnilnd_setup_rdma_buffer(tx, niov,
+ kiov, offset, mlen);
if (rc != 0)
GOTO(nak_get_req_rev, rc);
-
tx->tx_msg.gnm_u.putack.gnpam_src_cookie =
rxmsg->gnm_u.putreq.gnprm_cookie;
tx->tx_msg.gnm_u.putack.gnpam_dst_cookie = tx->tx_id.txe_cookie;
}
static void
-kgnilnd_reaper_poke_with_stick(unsigned long arg)
+kgnilnd_reaper_poke_with_stick(cfs_timer_cb_arg_t arg)
{
wake_up(&kgnilnd_data.kgn_reaper_waitq);
}
prepare_to_wait(&kgnilnd_data.kgn_reaper_waitq, &wait,
TASK_INTERRUPTIBLE);
spin_unlock(&kgnilnd_data.kgn_reaper_lock);
- setup_timer(&timer, kgnilnd_reaper_poke_with_stick,
- next_check_time);
+ cfs_timer_setup(&timer, kgnilnd_reaper_poke_with_stick,
+ next_check_time, 0);
mod_timer(&timer, (long) jiffies + timeout);
/* check flag variables before committing */
case GNILND_MSG_PUT_REQ:
case GNILND_MSG_GET_REQ_REV:
tx->tx_msg.gnm_u.putreq.gnprm_cookie = tx->tx_id.txe_cookie;
-
+ /* fallthrough */
case GNILND_MSG_PUT_ACK:
case GNILND_MSG_PUT_REQ_REV:
case GNILND_MSG_GET_ACK_REV:
new_ok -= atomic64_read(&dev->gnd_rdmaq_bytes_out);
atomic64_set(&dev->gnd_rdmaq_bytes_ok, new_ok);
- CDEBUG(D_NET, "resetting rdmaq bytes to %ld, deadline +%lu -> %lu, "
- "current out %ld\n",
- atomic64_read(&dev->gnd_rdmaq_bytes_ok), dead_bump, dev->gnd_rdmaq_deadline,
- atomic64_read(&dev->gnd_rdmaq_bytes_out));
+ CDEBUG(D_NET, "resetting rdmaq bytes to %lld, deadline +%lu -> %lu, current out %lld\n",
+ (s64)atomic64_read(&dev->gnd_rdmaq_bytes_ok), dead_bump, dev->gnd_rdmaq_deadline,
+ (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out));
}
spin_unlock(&dev->gnd_rdmaq_lock);
}
rx->grx_msg = msg;
rx->grx_conn = conn;
rx->grx_eager = 0;
- rx->grx_received = current_kernel_time();
+ ktime_get_ts64(&rx->grx_received);
if (CFS_FAIL_CHECK(CFS_FAIL_GNI_NET_LOOKUP)) {
rc = -ENONET;
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
kgnilnd_finalize_rx_done(tx, msg);
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
kgnilnd_complete_tx(tx, msg->gnm_u.completion.gncm_retval);
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
lnet_set_reply_msg_len(net->gnn_ni, tx->tx_lntmsg[1],
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
kgnilnd_finalize_rx_done(tx, msg);
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
kgnilnd_finalize_rx_done(tx, msg);
if (tx == NULL)
break;
- GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED ||
- tx->tx_buftype == GNILND_BUF_VIRT_MAPPED,
+ GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED,
"bad tx buftype %d", tx->tx_buftype);
kgnilnd_complete_tx(tx, msg->gnm_u.completion.gncm_retval);
} else {
GNIDBG_TX(log_retrans_level, tx,
"transient map failure #%d %d pages/%d bytes phys %u@%u "
- "virt %u@%llu "
"nq_map %d mdd# %d/%d GART %ld",
dev->gnd_map_attempt, tx->tx_phys_npages, tx->tx_nob,
dev->gnd_map_nphys, dev->gnd_map_physnop * PAGE_SIZE,
- dev->gnd_map_nvirt, dev->gnd_map_virtnob,
atomic_read(&dev->gnd_nq_map),
atomic_read(&dev->gnd_n_mdd), atomic_read(&dev->gnd_n_mdd_held),
atomic64_read(&dev->gnd_nbytes_map));
git://git.whamcloud.com / fs / lustre-release.git / blobdiff