+ } else if (payload_kiov != NULL) {
+ /* must checksum payload after header so receiver can
+ * compute partial header cksum before swab. Sadly
+ * this causes 2 rounds of kmap */
+ msg->kqm_cksum =
+ kqswnal_csum_kiov(msg->kqm_cksum, 0, payload_nob,
+ payload_niov, payload_kiov);
+ if (*kqswnal_tunables.kqn_inject_csum_error == 2) {
+ msg->kqm_cksum++;
+ *kqswnal_tunables.kqn_inject_csum_error = 0;
+ }
+ } else {
+ msg->kqm_cksum =
+ kqswnal_csum_iov(msg->kqm_cksum, 0, payload_nob,
+ payload_niov, payload_iov);
+ if (*kqswnal_tunables.kqn_inject_csum_error == 2) {
+ msg->kqm_cksum++;
+ *kqswnal_tunables.kqn_inject_csum_error = 0;
+ }
+#endif
+ }
+
+ } else if (payload_nob <= *kqswnal_tunables.kqn_tx_maxcontig) {
+ lnet_hdr_t *mhdr;
+ char *payload;
+ kqswnal_msg_t *msg = (kqswnal_msg_t *)ktx->ktx_buffer;
+
+ /* single frag copied into the pre-mapped buffer */
+ msg->kqm_magic = LNET_PROTO_QSW_MAGIC;
+ msg->kqm_version = QSWLND_PROTO_VERSION;
+ msg->kqm_type = QSWLND_MSG_IMMEDIATE;
+
+ mhdr = &msg->kqm_u.immediate.kqim_hdr;
+ payload = msg->kqm_u.immediate.kqim_payload;
+
+ *mhdr = *hdr;
+ nob = (payload - ktx->ktx_buffer) + payload_nob;
+
+ ep_nmd_subset(&ktx->ktx_frags[0], &ktx->ktx_ebuffer, 0, nob);
+
+ if (payload_kiov != NULL)
+ lnet_copy_kiov2flat(KQSW_TX_BUFFER_SIZE, payload, 0,
+ payload_niov, payload_kiov,
+ payload_offset, payload_nob);
+ else
+ lnet_copy_iov2flat(KQSW_TX_BUFFER_SIZE, payload, 0,
+ payload_niov, payload_iov,
+ payload_offset, payload_nob);
+#if KQSW_CKSUM
+ msg->kqm_nob = nob;
+ msg->kqm_cksum = 0;
+ msg->kqm_cksum = kqswnal_csum(~0, (char *)msg, nob);
+ if (*kqswnal_tunables.kqn_inject_csum_error == 1) {
+ msg->kqm_cksum++;
+ *kqswnal_tunables.kqn_inject_csum_error = 0;
+ }
+#endif
+ } else {
+ lnet_hdr_t *mhdr;
+ kqswnal_msg_t *msg = (kqswnal_msg_t *)ktx->ktx_buffer;
+
+ /* multiple frags: first is hdr in pre-mapped buffer */
+ msg->kqm_magic = LNET_PROTO_QSW_MAGIC;
+ msg->kqm_version = QSWLND_PROTO_VERSION;
+ msg->kqm_type = QSWLND_MSG_IMMEDIATE;
+
+ mhdr = &msg->kqm_u.immediate.kqim_hdr;
+ nob = offsetof(kqswnal_msg_t, kqm_u.immediate.kqim_payload);
+
+ *mhdr = *hdr;
+
+ ep_nmd_subset(&ktx->ktx_frags[0], &ktx->ktx_ebuffer, 0, nob);
+
+ if (payload_kiov != NULL)
+ rc = kqswnal_map_tx_kiov (ktx, payload_offset, payload_nob,
+ payload_niov, payload_kiov);
+ else
+ rc = kqswnal_map_tx_iov (ktx, payload_offset, payload_nob,
+ payload_niov, payload_iov);
+ if (rc != 0)
+ goto out;
+
+#if KQSW_CKSUM
+ msg->kqm_nob = nob + payload_nob;
+ msg->kqm_cksum = 0;
+ msg->kqm_cksum = kqswnal_csum(~0, (char *)msg, nob);
+
+ msg->kqm_cksum = (payload_kiov != NULL) ?
+ kqswnal_csum_kiov(msg->kqm_cksum,
+ payload_offset, payload_nob,
+ payload_niov, payload_kiov) :
+ kqswnal_csum_iov(msg->kqm_cksum,
+ payload_offset, payload_nob,
+ payload_niov, payload_iov);
+
+ if (*kqswnal_tunables.kqn_inject_csum_error == 1) {
+ msg->kqm_cksum++;
+ *kqswnal_tunables.kqn_inject_csum_error = 0;
+ }
+#endif
+ nob += payload_nob;
+ }
+
+ ktx->ktx_port = (nob <= KQSW_SMALLMSG) ?
+ EP_MSG_SVC_PORTALS_SMALL : EP_MSG_SVC_PORTALS_LARGE;