X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lnet%2Fklnds%2Fgnilnd%2Fgnilnd_cb.c;h=edd60cecc970b1da69a91bc253ac6c89fd69f02c;hb=3117913e21adb941e0ef3467ff7588d5e5fbc6e4;hp=a4d8f506d151b140fc06b2b41622ca179d2423ec;hpb=a35d4814f8444ec71c97377e1c276945fa31fc1b;p=fs%2Flustre-release.git diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index a4d8f50..edd60ce 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -87,9 +87,16 @@ kgnilnd_schedule_device(kgn_device_t *dev) } } -void kgnilnd_schedule_device_timer(unsigned long arg) +void kgnilnd_schedule_device_timer(cfs_timer_cb_arg_t data) { - kgn_device_t *dev = (kgn_device_t *) arg; + kgn_device_t *dev = cfs_from_timer(dev, data, gnd_map_timer); + + kgnilnd_schedule_device(dev); +} + +void kgnilnd_schedule_device_timer_rd(cfs_timer_cb_arg_t data) +{ + kgn_device_t *dev = cfs_from_timer(dev, data, gnd_rdmaq_timer); kgnilnd_schedule_device(dev); } @@ -277,7 +284,7 @@ kgnilnd_alloc_tx (void) if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ALLOC_TX)) return tx; - tx = kmem_cache_alloc(kgnilnd_data.kgn_tx_cache, GFP_ATOMIC); + tx = kmem_cache_zalloc(kgnilnd_data.kgn_tx_cache, GFP_ATOMIC); if (tx == NULL) { CERROR("failed to allocate tx\n"); return NULL; @@ -285,9 +292,6 @@ kgnilnd_alloc_tx (void) CDEBUG(D_MALLOC, "slab-alloced 'tx': %lu at %p.\n", sizeof(*tx), tx); - /* need this memset, cache alloc'd memory is not cleared */ - memset(tx, 0, sizeof(*tx)); - /* setup everything here to minimize time under the lock */ tx->tx_buftype = GNILND_BUF_NONE; tx->tx_msg.gnm_type = GNILND_MSG_NONE; @@ -304,8 +308,8 @@ kgnilnd_alloc_tx (void) #define _kgnilnd_cksum(seed, ptr, nob) csum_partial(ptr, nob, seed) /* we don't use offset as every one is passing a buffer reference that already - * includes the offset into the base address - - * see kgnilnd_setup_virt_buffer and kgnilnd_setup_immediate_buffer */ + * includes the offset into the base address. + */ static inline __u16 kgnilnd_cksum(void *ptr, size_t nob) { @@ -514,9 +518,9 @@ kgnilnd_nak_rdma(kgn_conn_t *conn, int rx_type, int error, __u64 cookie, lnet_ni kgnilnd_queue_tx(conn, tx); } -int +static int kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, - struct kvec *iov, struct bio_vec *kiov, + struct bio_vec *kiov, unsigned int offset, unsigned int nob) { kgn_msg_t *msg = &tx->tx_msg; @@ -529,16 +533,15 @@ kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, if (nob == 0) { tx->tx_buffer = NULL; - } else if (kiov != NULL) { + } else { - if ((niov > 0) && unlikely(niov > (nob/PAGE_SIZE))) { - niov = round_up(nob + offset + kiov->bv_offset, - PAGE_SIZE); - } + if (niov && niov > (nob >> PAGE_SHIFT)) + niov = DIV_ROUND_UP(nob + offset + kiov->bv_offset, + PAGE_SIZE); LASSERTF(niov > 0 && niov < GNILND_MAX_IMMEDIATE/PAGE_SIZE, - "bad niov %d msg %p kiov %p iov %p offset %d nob%d\n", - niov, msg, kiov, iov, offset, nob); + "bad niov %d msg %p kiov %p offset %d nob%d\n", + niov, msg, kiov, offset, nob); while (offset >= kiov->bv_len) { offset -= kiov->bv_len; @@ -594,29 +597,6 @@ kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, tx->tx_buftype = GNILND_BUF_IMMEDIATE_KIOV; tx->tx_nob = nob; - } else { - /* For now this is almost identical to kgnilnd_setup_virt_buffer, but we - * could "flatten" the payload into a single contiguous buffer ready - * for sending direct over an FMA if we ever needed to. */ - - LASSERT(niov > 0); - - while (offset >= iov->iov_len) { - offset -= iov->iov_len; - niov--; - iov++; - LASSERT(niov > 0); - } - - if (nob > iov->iov_len - offset) { - CERROR("Can't handle multiple vaddr fragments\n"); - return -EMSGSIZE; - } - - tx->tx_buffer = (void *)(((unsigned long)iov->iov_base) + offset); - - tx->tx_buftype = GNILND_BUF_IMMEDIATE; - tx->tx_nob = nob; } /* checksum payload early - it shouldn't be changing after lnd_send */ @@ -637,34 +617,6 @@ kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, } int -kgnilnd_setup_virt_buffer(kgn_tx_t *tx, - unsigned int niov, struct kvec *iov, - unsigned int offset, unsigned int nob) - -{ - LASSERT(nob > 0); - LASSERT(niov > 0); - LASSERT(tx->tx_buftype == GNILND_BUF_NONE); - - while (offset >= iov->iov_len) { - offset -= iov->iov_len; - niov--; - iov++; - LASSERT(niov > 0); - } - - if (nob > iov->iov_len - offset) { - CERROR("Can't handle multiple vaddr fragments\n"); - return -EMSGSIZE; - } - - tx->tx_buftype = GNILND_BUF_VIRT_UNMAPPED; - tx->tx_nob = nob; - tx->tx_buffer = (void *)(((unsigned long)iov->iov_base) + offset); - return 0; -} - -int kgnilnd_setup_phys_buffer(kgn_tx_t *tx, int nkiov, struct bio_vec *kiov, unsigned int offset, unsigned int nob) { @@ -779,21 +731,10 @@ error: static inline int kgnilnd_setup_rdma_buffer(kgn_tx_t *tx, unsigned int niov, - struct kvec *iov, struct bio_vec *kiov, + struct bio_vec *kiov, unsigned int offset, unsigned int nob) { - int rc; - - LASSERTF((iov == NULL) != (kiov == NULL), "iov 0x%p, kiov 0x%p, tx 0x%p," - " offset %d, nob %d, niov %d\n" - , iov, kiov, tx, offset, nob, niov); - - if (kiov != NULL) { - rc = kgnilnd_setup_phys_buffer(tx, niov, kiov, offset, nob); - } else { - rc = kgnilnd_setup_virt_buffer(tx, niov, iov, offset, nob); - } - return rc; + return kgnilnd_setup_phys_buffer(tx, niov, kiov, offset, nob); } /* kgnilnd_parse_lnet_rdma() @@ -934,7 +875,7 @@ kgnilnd_verify_rdma_cksum(kgn_tx_t *tx, __u16 rx_cksum, int put_len) kgnilnd_dump_blob(D_BUFFS, "RDMA payload", tx->tx_buffer, nob); } - /* fall through to dump log */ + /* fallthrough */ case 1: libcfs_debug_dumplog(); break; @@ -969,12 +910,6 @@ kgnilnd_mem_add_map_list(kgn_device_t *dev, kgn_tx_t *tx) dev->gnd_map_nphys++; dev->gnd_map_physnop += tx->tx_phys_npages; break; - - case GNILND_BUF_VIRT_MAPPED: - bytes = tx->tx_nob; - dev->gnd_map_nvirt++; - dev->gnd_map_virtnob += tx->tx_nob; - break; } if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK || @@ -1018,21 +953,16 @@ kgnilnd_mem_del_map_list(kgn_device_t *dev, kgn_tx_t *tx) dev->gnd_map_nphys--; dev->gnd_map_physnop -= tx->tx_phys_npages; break; - - case GNILND_BUF_VIRT_UNMAPPED: - bytes = tx->tx_nob; - dev->gnd_map_nvirt--; - dev->gnd_map_virtnob -= tx->tx_nob; - break; } if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK || tx->tx_msg.gnm_type == GNILND_MSG_GET_REQ) { atomic64_sub(bytes, &dev->gnd_rdmaq_bytes_out); LASSERTF(atomic64_read(&dev->gnd_rdmaq_bytes_out) >= 0, - "bytes_out negative! %ld\n", atomic64_read(&dev->gnd_rdmaq_bytes_out)); + "bytes_out negative! %lld\n", + (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out)); GNIDBG_TX(D_NETTRACE, tx, "rdma -- %d to %lld", - bytes, atomic64_read(&dev->gnd_rdmaq_bytes_out)); + bytes, (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out)); } atomic_dec(&dev->gnd_n_mdd); @@ -1076,7 +1006,6 @@ kgnilnd_map_buffer(kgn_tx_t *tx) case GNILND_BUF_IMMEDIATE: case GNILND_BUF_IMMEDIATE_KIOV: case GNILND_BUF_PHYS_MAPPED: - case GNILND_BUF_VIRT_MAPPED: return 0; case GNILND_BUF_PHYS_UNMAPPED: @@ -1089,41 +1018,16 @@ kgnilnd_map_buffer(kgn_tx_t *tx) * - this needs to turn into a non-fatal error soon to allow * GART resource, etc starvation handling */ if (rrc != GNI_RC_SUCCESS) { - GNIDBG_TX(D_NET, tx, "Can't map %d pages: dev %d " - "phys %u pp %u, virt %u nob %llu", + GNIDBG_TX(D_NET, tx, + "Can't map %d pages: dev %d phys %u pp %u", tx->tx_phys_npages, dev->gnd_id, - dev->gnd_map_nphys, dev->gnd_map_physnop, - dev->gnd_map_nvirt, dev->gnd_map_virtnob); + dev->gnd_map_nphys, dev->gnd_map_physnop); RETURN(rrc == GNI_RC_ERROR_RESOURCE ? -ENOMEM : -EINVAL); } tx->tx_buftype = GNILND_BUF_PHYS_MAPPED; kgnilnd_mem_add_map_list(dev, tx); return 0; - - case GNILND_BUF_VIRT_UNMAPPED: - rrc = kgnilnd_mem_register(dev->gnd_handle, - (__u64)tx->tx_buffer, tx->tx_nob, - NULL, flags, &tx->tx_map_key); - if (rrc != GNI_RC_SUCCESS) { - GNIDBG_TX(D_NET, tx, "Can't map %u bytes: dev %d " - "phys %u pp %u, virt %u nob %llu", - tx->tx_nob, dev->gnd_id, - dev->gnd_map_nphys, dev->gnd_map_physnop, - dev->gnd_map_nvirt, dev->gnd_map_virtnob); - RETURN(rrc == GNI_RC_ERROR_RESOURCE ? -ENOMEM : -EINVAL); - } - - tx->tx_buftype = GNILND_BUF_VIRT_MAPPED; - kgnilnd_mem_add_map_list(dev, tx); - if (tx->tx_msg.gnm_type == GNILND_MSG_PUT_ACK || - tx->tx_msg.gnm_type == GNILND_MSG_GET_REQ) { - atomic64_add(tx->tx_nob, &dev->gnd_rdmaq_bytes_out); - GNIDBG_TX(D_NETTRACE, tx, "rdma ++ %d to %ld\n", - tx->tx_nob, atomic64_read(&dev->gnd_rdmaq_bytes_out)); - } - - return 0; } } @@ -1167,8 +1071,6 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error) /* code below relies on +1 relationship ... */ BUILD_BUG_ON(GNILND_BUF_PHYS_MAPPED != (GNILND_BUF_PHYS_UNMAPPED + 1)); - BUILD_BUG_ON(GNILND_BUF_VIRT_MAPPED != - (GNILND_BUF_VIRT_UNMAPPED + 1)); switch (tx->tx_buftype) { default: @@ -1177,7 +1079,6 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error) case GNILND_BUF_NONE: case GNILND_BUF_IMMEDIATE: case GNILND_BUF_PHYS_UNMAPPED: - case GNILND_BUF_VIRT_UNMAPPED: break; case GNILND_BUF_IMMEDIATE_KIOV: if (tx->tx_phys != NULL) { @@ -1191,7 +1092,6 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error) break; case GNILND_BUF_PHYS_MAPPED: - case GNILND_BUF_VIRT_MAPPED: LASSERT(tx->tx_conn != NULL); dev = tx->tx_conn->gnc_device; @@ -1761,7 +1661,8 @@ kgnilnd_queue_tx(kgn_conn_t *conn, kgn_tx_t *tx) /* it was sent, break out of switch to avoid default case of queueing */ break; } - /* needs to queue to try again, so fall through to default case */ + /* needs to queue to try again, so... */ + /* fall through... */ case GNILND_MSG_NOOP: /* Just make sure this goes out first for this conn */ add_tail = 0; @@ -2183,7 +2084,7 @@ kgnilnd_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) goto out; } rc = kgnilnd_setup_rdma_buffer(tx, lntmsg->msg_md->md_niov, - NULL, lntmsg->msg_md->md_kiov, + lntmsg->msg_md->md_kiov, 0, lntmsg->msg_md->md_length); if (rc != 0) { CERROR("unable to setup buffer: %d\n", rc); @@ -2227,7 +2128,7 @@ kgnilnd_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) goto out; } - rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL, + rc = kgnilnd_setup_rdma_buffer(tx, niov, kiov, offset, nob); if (rc != 0) { kgnilnd_tx_done(tx, rc); @@ -2257,7 +2158,7 @@ kgnilnd_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg) goto out; } - rc = kgnilnd_setup_immediate_buffer(tx, niov, NULL, kiov, offset, nob); + rc = kgnilnd_setup_immediate_buffer(tx, niov, kiov, offset, nob); if (rc != 0) { kgnilnd_tx_done(tx, rc); goto out; @@ -2310,7 +2211,7 @@ kgnilnd_setup_rdma(struct lnet_ni *ni, kgn_rx_t *rx, struct lnet_msg *lntmsg, in if (rc != 0) goto failed_1; - rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL, kiov, offset, nob); + rc = kgnilnd_setup_rdma_buffer(tx, niov, kiov, offset, nob); if (rc != 0) goto failed_1; @@ -2501,7 +2402,7 @@ kgnilnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg, case 2: kgnilnd_dump_blob(D_BUFFS, "bad payload checksum", &rxmsg[1], rxmsg->gnm_payload_len); - /* fall through to dump */ + /* fallthrough */ case 1: libcfs_debug_dumplog(); break; @@ -2516,16 +2417,10 @@ kgnilnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg, } } - if (kiov != NULL) - lnet_copy_flat2kiov( - niov, kiov, offset, - *kgnilnd_tunables.kgn_max_immediate, - &rxmsg[1], 0, mlen); - else - lnet_copy_flat2iov( - niov, NULL, offset, - *kgnilnd_tunables.kgn_max_immediate, - &rxmsg[1], 0, mlen); + lnet_copy_flat2kiov( + niov, kiov, offset, + *kgnilnd_tunables.kgn_max_immediate, + &rxmsg[1], 0, mlen); kgnilnd_consume_rx(rx); lnet_finalize(lntmsg, 0); @@ -2557,7 +2452,7 @@ kgnilnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg, GOTO(nak_put_req, rc); } - rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL, + rc = kgnilnd_setup_rdma_buffer(tx, niov, kiov, offset, mlen); if (rc != 0) { GOTO(nak_put_req, rc); @@ -2618,13 +2513,11 @@ nak_put_req: if (rc != 0) GOTO(nak_get_req_rev, rc); - - rc = kgnilnd_setup_rdma_buffer(tx, niov, NULL, + rc = kgnilnd_setup_rdma_buffer(tx, niov, kiov, offset, mlen); if (rc != 0) GOTO(nak_get_req_rev, rc); - tx->tx_msg.gnm_u.putack.gnpam_src_cookie = rxmsg->gnm_u.putreq.gnprm_cookie; tx->tx_msg.gnm_u.putack.gnpam_dst_cookie = tx->tx_id.txe_cookie; @@ -3023,7 +2916,7 @@ kgnilnd_update_reaper_timeout(long timeout) } static void -kgnilnd_reaper_poke_with_stick(unsigned long arg) +kgnilnd_reaper_poke_with_stick(cfs_timer_cb_arg_t arg) { wake_up(&kgnilnd_data.kgn_reaper_waitq); } @@ -3066,8 +2959,8 @@ kgnilnd_reaper(void *arg) prepare_to_wait(&kgnilnd_data.kgn_reaper_waitq, &wait, TASK_INTERRUPTIBLE); spin_unlock(&kgnilnd_data.kgn_reaper_lock); - setup_timer(&timer, kgnilnd_reaper_poke_with_stick, - next_check_time); + cfs_timer_setup(&timer, kgnilnd_reaper_poke_with_stick, + next_check_time, 0); mod_timer(&timer, (long) jiffies + timeout); /* check flag variables before committing */ @@ -3844,7 +3737,7 @@ kgnilnd_process_fmaq(kgn_conn_t *conn) case GNILND_MSG_PUT_REQ: case GNILND_MSG_GET_REQ_REV: tx->tx_msg.gnm_u.putreq.gnprm_cookie = tx->tx_id.txe_cookie; - + /* fallthrough */ case GNILND_MSG_PUT_ACK: case GNILND_MSG_PUT_REQ_REV: case GNILND_MSG_GET_ACK_REV: @@ -3924,10 +3817,9 @@ kgnilnd_process_rdmaq(kgn_device_t *dev) new_ok -= atomic64_read(&dev->gnd_rdmaq_bytes_out); atomic64_set(&dev->gnd_rdmaq_bytes_ok, new_ok); - CDEBUG(D_NET, "resetting rdmaq bytes to %ld, deadline +%lu -> %lu, " - "current out %ld\n", - atomic64_read(&dev->gnd_rdmaq_bytes_ok), dead_bump, dev->gnd_rdmaq_deadline, - atomic64_read(&dev->gnd_rdmaq_bytes_out)); + CDEBUG(D_NET, "resetting rdmaq bytes to %lld, deadline +%lu -> %lu, current out %lld\n", + (s64)atomic64_read(&dev->gnd_rdmaq_bytes_ok), dead_bump, dev->gnd_rdmaq_deadline, + (s64)atomic64_read(&dev->gnd_rdmaq_bytes_out)); } spin_unlock(&dev->gnd_rdmaq_lock); } @@ -4218,7 +4110,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) rx->grx_msg = msg; rx->grx_conn = conn; rx->grx_eager = 0; - rx->grx_received = current_kernel_time(); + ktime_get_ts64(&rx->grx_received); if (CFS_FAIL_CHECK(CFS_FAIL_GNI_NET_LOOKUP)) { rc = -ENONET; @@ -4479,8 +4371,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); kgnilnd_finalize_rx_done(tx, msg); @@ -4498,8 +4389,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); kgnilnd_complete_tx(tx, msg->gnm_u.completion.gncm_retval); @@ -4511,8 +4401,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); lnet_set_reply_msg_len(net->gnn_ni, tx->tx_lntmsg[1], @@ -4526,8 +4415,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); kgnilnd_finalize_rx_done(tx, msg); @@ -4540,8 +4428,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); kgnilnd_finalize_rx_done(tx, msg); @@ -4553,8 +4440,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (tx == NULL) break; - GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED || - tx->tx_buftype == GNILND_BUF_VIRT_MAPPED, + GNITX_ASSERTF(tx, tx->tx_buftype == GNILND_BUF_PHYS_MAPPED, "bad tx buftype %d", tx->tx_buftype); kgnilnd_complete_tx(tx, msg->gnm_u.completion.gncm_retval); @@ -4857,11 +4743,9 @@ kgnilnd_process_mapped_tx(kgn_device_t *dev) } else { GNIDBG_TX(log_retrans_level, tx, "transient map failure #%d %d pages/%d bytes phys %u@%u " - "virt %u@%llu " "nq_map %d mdd# %d/%d GART %ld", dev->gnd_map_attempt, tx->tx_phys_npages, tx->tx_nob, dev->gnd_map_nphys, dev->gnd_map_physnop * PAGE_SIZE, - dev->gnd_map_nvirt, dev->gnd_map_virtnob, atomic_read(&dev->gnd_nq_map), atomic_read(&dev->gnd_n_mdd), atomic_read(&dev->gnd_n_mdd_held), atomic64_read(&dev->gnd_nbytes_map));