/* # tx callbacks outstanding */
short tx_sending;
/* queued for sending */
- short tx_queued;
+ unsigned long tx_queued:1,
/* waiting for peer_ni */
- short tx_waiting;
+ tx_waiting:1,
+ /* force RDMA */
+ tx_gpu:1;
/* LNET completion status */
int tx_status;
/* health status of the transmit */
#define KIBLND_UNMAP_ADDR_SET(p, m, a) do {} while (0)
#define KIBLND_UNMAP_ADDR(p, m, a) (a)
-static inline int kiblnd_dma_map_sg(struct kib_hca_dev *hdev,
- struct scatterlist *sg, int nents,
- enum dma_data_direction direction)
+static inline
+int kiblnd_dma_map_sg(struct kib_hca_dev *hdev, struct kib_tx *tx)
{
- int count;
+ struct scatterlist *sg = tx->tx_frags;
+ int nents = tx->tx_nfrags;
+ enum dma_data_direction direction = tx->tx_dmadir;
- count = lnet_rdma_map_sg_attrs(hdev->ibh_ibdev->dma_device,
- sg, nents, direction);
-
- if (count != 0)
- return count;
+ if (tx->tx_gpu)
+ return lnet_rdma_map_sg_attrs(hdev->ibh_ibdev->dma_device,
+ sg, nents, direction);
return ib_dma_map_sg(hdev->ibh_ibdev, sg, nents, direction);
}
-static inline void kiblnd_dma_unmap_sg(struct kib_hca_dev *hdev,
- struct scatterlist *sg, int nents,
- enum dma_data_direction direction)
+static inline
+void kiblnd_dma_unmap_sg(struct kib_hca_dev *hdev, struct kib_tx *tx)
{
- int count;
-
- count = lnet_rdma_unmap_sg(hdev->ibh_ibdev->dma_device,
- sg, nents, direction);
- if (count != 0)
- return;
+ struct scatterlist *sg = tx->tx_frags;
+ int nents = tx->tx_nfrags;
+ enum dma_data_direction direction = tx->tx_dmadir;
- ib_dma_unmap_sg(hdev->ibh_ibdev, sg, nents, direction);
+ if (tx->tx_gpu)
+ lnet_rdma_unmap_sg(hdev->ibh_ibdev->dma_device,
+ sg, nents, direction);
+ else
+ ib_dma_unmap_sg(hdev->ibh_ibdev, sg, nents, direction);
}
#ifndef HAVE_IB_SG_DMA_ADDRESS
kiblnd_fmr_pool_unmap(&tx->tx_fmr, tx->tx_status);
if (tx->tx_nfrags != 0) {
- kiblnd_dma_unmap_sg(tx->tx_pool->tpo_hdev,
- tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir);
+ kiblnd_dma_unmap_sg(tx->tx_pool->tpo_hdev, tx);
tx->tx_nfrags = 0;
}
}
tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
tx->tx_nfrags = nfrags;
- rd->rd_nfrags = kiblnd_dma_map_sg(hdev, tx->tx_frags,
- tx->tx_nfrags, tx->tx_dmadir);
-
+ rd->rd_nfrags = kiblnd_dma_map_sg(hdev, tx);
for (i = 0, nob = 0; i < rd->rd_nfrags; i++) {
rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len(
hdev->ibh_ibdev, &tx->tx_frags[i]);
int prev = dstidx;
if (srcidx >= srcrd->rd_nfrags) {
- CERROR("Src buffer exhausted: %d frags\n", srcidx);
+ CERROR("Src buffer exhausted: %d frags %px\n",
+ srcidx, tx);
rc = -EPROTO;
break;
}
struct bio_vec *payload_kiov = lntmsg->msg_kiov;
unsigned int payload_offset = lntmsg->msg_offset;
unsigned int payload_nob = lntmsg->msg_len;
+ bool gpu;
struct kib_msg *ibmsg;
struct kib_rdma_desc *rd;
- struct kib_tx *tx;
- int nob;
- int rc;
+ struct kib_tx *tx;
+ int nob;
+ int rc;
/* NB 'private' is different depending on what we're sending.... */
return -ENOMEM;
}
ibmsg = tx->tx_msg;
+ gpu = (lntmsg->msg_md->md_flags & LNET_MD_FLAG_GPU);
switch (type) {
default:
LBUG();
return (-EIO);
- case LNET_MSG_ACK:
- LASSERT (payload_nob == 0);
- break;
+ case LNET_MSG_ACK:
+ LASSERT(payload_nob == 0);
+ break;
- case LNET_MSG_GET:
- if (routing || target_is_router)
- break; /* send IMMEDIATE */
+ case LNET_MSG_GET:
+ if (routing || target_is_router)
+ break; /* send IMMEDIATE */
- /* is the REPLY message too small for RDMA? */
+ /* is the REPLY message too small for RDMA? */
nob = offsetof(struct kib_msg, ibm_u.immediate.ibim_payload[lntmsg->msg_md->md_length]);
- if (nob <= IBLND_MSG_SIZE && !lntmsg->msg_rdma_force)
+ if (nob <= IBLND_MSG_SIZE && !gpu)
break; /* send IMMEDIATE */
rd = &ibmsg->ibm_u.get.ibgm_rd;
+ tx->tx_gpu = !!gpu;
rc = kiblnd_setup_rd_kiov(ni, tx, rd,
lntmsg->msg_md->md_niov,
lntmsg->msg_md->md_kiov,
ibmsg->ibm_u.get.ibgm_cookie = tx->tx_cookie;
lnet_hdr_to_nid4(hdr, &ibmsg->ibm_u.get.ibgm_hdr);
- kiblnd_init_tx_msg(ni, tx, IBLND_MSG_GET_REQ, nob);
+ kiblnd_init_tx_msg(ni, tx, IBLND_MSG_GET_REQ, nob);
- tx->tx_lntmsg[1] = lnet_create_reply_msg(ni, lntmsg);
+ tx->tx_lntmsg[1] = lnet_create_reply_msg(ni, lntmsg);
if (tx->tx_lntmsg[1] == NULL) {
CERROR("Can't create reply for GET -> %s\n",
libcfs_nidstr(&target->nid));
case LNET_MSG_REPLY:
case LNET_MSG_PUT:
/* Is the payload small enough not to need RDMA? */
- nob = offsetof(struct kib_msg, ibm_u.immediate.ibim_payload[payload_nob]);
- if (nob <= IBLND_MSG_SIZE && !lntmsg->msg_rdma_force)
- break; /* send IMMEDIATE */
+ nob = offsetof(struct kib_msg,
+ ibm_u.immediate.ibim_payload[payload_nob]);
+ if (nob <= IBLND_MSG_SIZE && !gpu)
+ break; /* send IMMEDIATE */
+
+ tx->tx_gpu = gpu;
rc = kiblnd_setup_rd_kiov(ni, tx, tx->tx_rd,
payload_niov, payload_kiov,
struct bio_vec *kiov = lntmsg->msg_kiov;
unsigned int offset = lntmsg->msg_offset;
unsigned int nob = lntmsg->msg_len;
+ struct lnet_libmd *payload_md = lntmsg->msg_md;
struct kib_tx *tx;
int rc;
goto failed_0;
}
+ tx->tx_gpu = !!(payload_md->md_flags & LNET_MD_FLAG_GPU);
if (nob == 0)
rc = 0;
else
case IBLND_MSG_PUT_REQ: {
struct kib_msg *txmsg;
struct kib_rdma_desc *rd;
- ibprm_cookie = rxmsg->ibm_u.putreq.ibprm_cookie;
+ struct lnet_libmd *payload_md = lntmsg->msg_md;
+ ibprm_cookie = rxmsg->ibm_u.putreq.ibprm_cookie;
if (mlen == 0) {
lnet_finalize(lntmsg, 0);
kiblnd_send_completion(rx->rx_conn, IBLND_MSG_PUT_NAK,
}
tx = kiblnd_get_idle_tx(ni, conn->ibc_peer->ibp_nid);
- if (tx == NULL) {
- CERROR("Can't allocate tx for %s\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- /* Not replying will break the connection */
- rc = -ENOMEM;
- break;
- }
+ if (tx == NULL) {
+ CERROR("Can't allocate tx for %s\n",
+ libcfs_nid2str(conn->ibc_peer->ibp_nid));
+ /* Not replying will break the connection */
+ rc = -ENOMEM;
+ break;
+ }
+ tx->tx_gpu = !!(payload_md->md_flags & LNET_MD_FLAG_GPU);
txmsg = tx->tx_msg;
rd = &txmsg->ibm_u.putack.ibpam_rd;
rc = kiblnd_setup_rd_kiov(ni, tx, rd,