From 6d5afffb50fe524ab40308e6f2a2dbda3851b8b5 Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Thu, 5 Jun 2025 01:49:03 +0700 Subject: [PATCH] LU-18925 libcfs: prefer kmap_local_page Prefer kmap_local* in favor of kmap and kmap_atomic in preparation of conversion to struct folio and kmap_local_folio Provide ll_kmap_local_page alias to maintain mapping to kmap_atomic use in older kernels. Provide kmap_local_page mapping to kmap for older kernels Signed-off-by: Shaun Tancheff Change-Id: I89ec7de9b6317a72df739500c0c3a942568e5eeb --- config/lustre-libcfs.m4 | 43 ++++++++++++++++++++ libcfs/libcfs/tracefile.c | 8 ++-- lnet/klnds/gnilnd/gnilnd_cb.c | 16 ++++---- lnet/klnds/socklnd/socklnd_lib.c | 41 ++++++++++--------- lnet/lnet/lib-move.c | 82 ++++++++++++++++++++++---------------- lnet/lnet/lnet-crypto.c | 4 +- lustre/include/lu_object.h | 2 +- lustre/include/obd_support.h | 7 +++- lustre/llite/dir.c | 2 - lustre/llite/file.c | 8 ++-- lustre/llite/rw26.c | 4 +- lustre/lmv/lmv_obd.c | 15 ++++--- lustre/lov/lov_page.c | 4 +- lustre/mdc/mdc_request.c | 28 +++++++------ lustre/mdd/mdd_object.c | 6 +-- lustre/mdt/mdt_io.c | 4 +- lustre/mgc/mgc_request.c | 4 +- lustre/mgc/mgc_request_server.c | 4 +- lustre/mgs/mgs_nids.c | 7 ++-- lustre/obdclass/dt_object.c | 21 +++++----- lustre/obdclass/integrity.c | 14 ++++--- lustre/obdclass/jobid.c | 4 +- lustre/obdecho/echo.c | 33 ++++++++------- lustre/obdecho/echo_client.c | 8 ++-- lustre/osc/osc_request.c | 36 +++++++++-------- lustre/osd-ldiskfs/osd_integrity.c | 4 +- lustre/osd-ldiskfs/osd_io.c | 11 +++-- lustre/osd-zfs/osd_io.c | 6 ++- lustre/osp/osp_internal.h | 2 +- lustre/osp/osp_object.c | 19 ++++----- lustre/ptlrpc/sec_plain.c | 4 +- lustre/quota/qsd_reint.c | 5 ++- lustre/target/tgt_handler.c | 48 ++++++++++++---------- 33 files changed, 298 insertions(+), 206 deletions(-) diff --git a/config/lustre-libcfs.m4 b/config/lustre-libcfs.m4 index f5609a8..2f58ced 100644 --- a/config/lustre-libcfs.m4 +++ b/config/lustre-libcfs.m4 @@ -2130,6 +2130,47 @@ AC_DEFUN([LIBCFS_SEC_RELEASE_SECCTX], [ ]) # LIBCFS_SEC_RELEASE_SECCTX # +# LIBCFS_HAVE_KMAP_LOCAL +# +# Linux commit v5.10-rc2-80-gf3ba3c710ac5 +# mm/highmem: Provide kmap_local* +# +AC_DEFUN([LIBCFS_SRC_HAVE_KMAP_LOCAL], [ + LB2_LINUX_TEST_SRC([kmap_local_page], [ + #include + ],[ + struct page *pg = NULL; + void *kaddr = kmap_local_page(pg); + + kunmap_local(kaddr); + ],[-Werror]) +]) +AC_DEFUN([LIBCFS_HAVE_KMAP_LOCAL], [ + LB2_MSG_LINUX_TEST_RESULT([if 'kmap_local*' are available], + [kmap_local_page], [ + AC_DEFINE(HAVE_KMAP_LOCAL, 1, + [kmap_local_* functions are available]) + ## Map ll_k[un]map_local* to kmap_local* + AC_DEFINE([ll_kmap_local_page(p)], [kmap_local_page(p)], + [alias for kmap_local_page()]) + AC_DEFINE([ll_kunmap_local(kaddr)], [kunmap_local((kaddr))], + [alias for kunmap_local()]) + ],[ + ## Map k[un]map_local* to k[un]map + AC_DEFINE([kmap_local_page(p)], [kmap(p)], + [need kmap_local_page()]) + AC_DEFINE([kunmap_local(kaddr)], [kunmap(kmap_to_page(kaddr))], + [need kunmap_local()]) + ## Map ll_k[un]map_local* to k[un]map_atomic for older kernels + ## that do not have k[un]map_local* available. + AC_DEFINE([ll_kmap_local_page(p)], [kmap_atomic(p)], + [need kmap_local_page map to atomic]) + AC_DEFINE([ll_kunmap_local(kaddr)], [kunmap_atomic((kaddr))], + [need kunmap_local map to atomic]) + ]) +]) # LIBCFS_HAVE_KMAP_LOCAL + +# # LIBCFS_HAVE_KFREE_SENSITIVE # # kernel v5.10-rc1~3 @@ -2650,6 +2691,7 @@ AC_DEFUN([LIBCFS_PROG_LINUX_SRC], [ LIBCFS_SRC_KEY_NEED_UNLINK LIBCFS_SRC_SEC_RELEASE_SECCTX # 5.10 + LIBCFS_SRC_HAVE_KMAP_LOCAL LIBCFS_SRC_HAVE_KFREE_SENSITIVE LIBCFS_SRC_HAVE_CRYPTO_SHA2_HEADER LIBCFS_SRC_HAVE_LIST_CMP_FUNC_T @@ -2807,6 +2849,7 @@ AC_DEFUN([LIBCFS_PROG_LINUX_RESULTS], [ LIBCFS_KEY_NEED_UNLINK LIBCFS_SEC_RELEASE_SECCTX # 5.10 + LIBCFS_HAVE_KMAP_LOCAL LIBCFS_HAVE_KFREE_SENSITIVE LIBCFS_HAVE_CRYPTO_SHA2_HEADER LIBCFS_HAVE_LIST_CMP_FUNC_T diff --git a/libcfs/libcfs/tracefile.c b/libcfs/libcfs/tracefile.c index c38f7d1..6d43e5ff 100644 --- a/libcfs/libcfs/tracefile.c +++ b/libcfs/libcfs/tracefile.c @@ -924,9 +924,9 @@ int cfs_tracefile_dump_all_pages(char *filename) __LASSERT_TAGE_INVARIANT(tage); - buf = kmap(tage->page); + buf = kmap_local_page(tage->page); rc = cfs_kernel_write(filp, buf, tage->used, &filp->f_pos); - kunmap(tage->page); + kunmap_local(buf); if (rc != (int)tage->used) { pr_warn("Lustre: wanted to write %u but wrote %d\n", tage->used, rc); @@ -1191,10 +1191,10 @@ static int tracefiled(void *arg) else if (f_pos > i_size_read(de->d_inode)) f_pos = i_size_read(de->d_inode); - buf = kmap(tage->page); + buf = kmap_local_page(tage->page); rc = cfs_kernel_write(filp, buf, tage->used, &f_pos); - kunmap(tage->page); + kunmap_local(buf); if (rc != (int)tage->used) { pr_warn("Lustre: wanted to write %u but wrote %d\n", tage->used, rc); diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index a1d7255..e4045a6 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -382,6 +382,8 @@ kgnilnd_cksum_kiov(unsigned int nkiov, struct bio_vec *kiov, vunmap(addr); } else { do { + void *kaddr; + fraglen = min(kiov->bv_len - offset, nob); /* make dang sure we don't send a bogus checksum if somehow we get @@ -392,9 +394,8 @@ kgnilnd_cksum_kiov(unsigned int nkiov, struct bio_vec *kiov, "odd fraglen %u on nkiov %d, nob %u bv_len %u offset %u kiov 0x%p\n", fraglen, nkiov, nob, kiov->bv_len, offset, kiov); - - addr = (void *)kmap(kiov->bv_page) + kiov->bv_offset + - offset; + kaddr = kmap_local_page(kiov->bv_page) + addr = kaddr + kiov->bv_offset + offset; tmpck = _kgnilnd_cksum(cksum, addr, fraglen); CDEBUG(D_BUFFS, @@ -407,7 +408,7 @@ kgnilnd_cksum_kiov(unsigned int nkiov, struct bio_vec *kiov, if (dump_blob) kgnilnd_dump_blob(D_BUFFS, "kiov cksum", addr, fraglen); - kunmap(kiov->bv_page); + kunmap_local(kaddr); kiov++; nkiov--; @@ -560,10 +561,11 @@ kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, /* hijack tx_phys for the later unmap */ if (niov == 1) { + void *kaddr = kmap_local_page(tx->tx_imm_pages[0]); + /* tx->phyx being equal to NULL is the signal for unmap to discern between kmap and vmap */ tx->tx_phys = NULL; - tx->tx_buffer = (void *)kmap(tx->tx_imm_pages[0]) + - kiov[0].bv_offset + offset; + tx->tx_buffer = kaddr + kiov[0].bv_offset + offset; atomic_inc(&kgnilnd_data.kgn_nkmap_short); GNIDBG_TX(D_NET, tx, "kmapped page for %d bytes for kiov 0x%p, buffer 0x%p", nob, kiov, tx->tx_buffer); @@ -1074,7 +1076,7 @@ kgnilnd_unmap_buffer(kgn_tx_t *tx, int error) if (tx->tx_phys != NULL) { vunmap(tx->tx_phys); } else if (tx->tx_phys == NULL && tx->tx_buffer != NULL) { - kunmap(tx->tx_imm_pages[0]); + kunmap_local(tx->tx_buffer); } /* clear to prevent kgnilnd_free_tx from thinking * this is a RDMA descriptor */ diff --git a/lnet/klnds/socklnd/socklnd_lib.c b/lnet/klnds/socklnd/socklnd_lib.c index 2b9dbc5..deeb33a 100644 --- a/lnet/klnds/socklnd/socklnd_lib.c +++ b/lnet/klnds/socklnd/socklnd_lib.c @@ -156,8 +156,9 @@ ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx, int i; for (nob = i = 0; i < niov; i++) { - scratchiov[i].iov_base = kmap(kiov[i].bv_page) + - kiov[i].bv_offset; + void *kaddr = kmap_local_page(kiov[i].bv_page); + + scratchiov[i].iov_base = kaddr + kiov[i].bv_offset; nob += scratchiov[i].iov_len = kiov[i].bv_len; } @@ -167,8 +168,8 @@ ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx, rc = kernel_sendmsg(sock, &msg, scratchiov, niov, nob); - for (i = 0; i < niov; i++) - kunmap(kiov[i].bv_page); + for (i = niov; i > 0; ) + kunmap_local(scratchiov[--i].iov_base); } return rc; } @@ -323,15 +324,15 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn, struct page **pages, * or leave them alone. */ addr = ksocknal_lib_kiov_vmap(kiov, niov, scratchiov, pages); - if (addr != NULL) { + if (addr) { nob = scratchiov[0].iov_len; n = 1; - } else { for (nob = i = 0; i < niov; i++) { + void *kaddr = kmap_local_page(kiov[i].bv_page); + + scratchiov[i].iov_base = kaddr + kiov[i].bv_offset; nob += scratchiov[i].iov_len = kiov[i].bv_len; - scratchiov[i].iov_base = kmap(kiov[i].bv_page) + - kiov[i].bv_offset; } n = niov; } @@ -341,8 +342,15 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn, struct page **pages, rc = kernel_recvmsg(conn->ksnc_sock, &msg, scratchiov, n, nob, MSG_DONTWAIT); + if (!addr) { + for (i = niov; i > 0; ) + kunmap_local(scratchiov[--i].iov_base); + } + if (conn->ksnc_msg.ksm_csum != 0) { for (i = 0, sum = rc; sum > 0; i++, sum -= fragnob) { + void *kaddr = kmap_local_page(kiov[i].bv_page); + LASSERT(i < niov); /* Dang! have to kmap again because I have nowhere to @@ -350,7 +358,7 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn, struct page **pages, * page is still mapped, the kernel just bumps the map * count and returns me the address it stashed. */ - base = kmap(kiov[i].bv_page) + kiov[i].bv_offset; + base = kaddr + kiov[i].bv_offset; fragnob = kiov[i].bv_len; if (fragnob > sum) fragnob = sum; @@ -358,16 +366,12 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn, struct page **pages, conn->ksnc_rx_csum = ksocknal_csum(conn->ksnc_rx_csum, base, fragnob); - kunmap(kiov[i].bv_page); + kunmap_local(kaddr); } } - if (addr != NULL) { + if (addr) ksocknal_lib_kiov_vunmap(addr); - } else { - for (i = 0; i < niov; i++) - kunmap(kiov[i].bv_page); - } return rc; } @@ -389,12 +393,11 @@ ksocknal_lib_csum_tx(struct ksock_tx *tx) tx->tx_hdr.iov_len); for (i = 0; i < tx->tx_nkiov; i++) { - base = kmap(tx->tx_kiov[i].bv_page) + - tx->tx_kiov[i].bv_offset; + void *kaddr = kmap_local_page(tx->tx_kiov[i].bv_page); + base = kaddr + tx->tx_kiov[i].bv_offset; csum = ksocknal_csum(csum, base, tx->tx_kiov[i].bv_len); - - kunmap(tx->tx_kiov[i].bv_page); + kunmap_local(kaddr); } if (*ksocknal_tunables.ksnd_inject_csum_error) { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 49d2c6c..586eba3 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -362,9 +362,11 @@ lnet_copy_kiov2kiov(unsigned int ndiov, struct bio_vec *diov, unsigned int nob) { /* NB diov, siov are READ-ONLY */ - unsigned int this_nob; - char *daddr = NULL; - char *saddr = NULL; + unsigned int this_nob; + char *daddr = NULL; + char *saddr = NULL; + void *dkaddr; + void *skaddr; if (nob == 0) return; @@ -394,12 +396,14 @@ lnet_copy_kiov2kiov(unsigned int ndiov, struct bio_vec *diov, siov->bv_len - soffset, nob); - if (daddr == NULL) - daddr = ((char *)kmap(diov->bv_page)) + - diov->bv_offset + doffset; - if (saddr == NULL) - saddr = ((char *)kmap(siov->bv_page)) + - siov->bv_offset + soffset; + if (!daddr) { + dkaddr = kmap_local_page(diov->bv_page); + daddr = dkaddr + diov->bv_offset + doffset; + } + if (!saddr) { + skaddr = kmap_local_page(siov->bv_page); + saddr = skaddr + siov->bv_offset + soffset; + } /* Vanishing risk of kmap deadlock when mapping 2 pages. * However in practice at least one of the kiovs will be mapped @@ -416,7 +420,8 @@ lnet_copy_kiov2kiov(unsigned int ndiov, struct bio_vec *diov, daddr += this_nob; doffset += this_nob; } else { - kunmap(diov->bv_page); + kunmap_local(dkaddr); + dkaddr = NULL; daddr = NULL; diov++; ndiov--; @@ -427,7 +432,8 @@ lnet_copy_kiov2kiov(unsigned int ndiov, struct bio_vec *diov, saddr += this_nob; soffset += this_nob; } else { - kunmap(siov->bv_page); + kunmap_local(skaddr); + skaddr = NULL; saddr = NULL; siov++; nsiov--; @@ -435,22 +441,23 @@ lnet_copy_kiov2kiov(unsigned int ndiov, struct bio_vec *diov, } } while (nob > 0); - if (daddr != NULL) - kunmap(diov->bv_page); - if (saddr != NULL) - kunmap(siov->bv_page); + if (daddr) + kunmap_local(dkaddr); + if (saddr) + kunmap_local(skaddr); } EXPORT_SYMBOL(lnet_copy_kiov2kiov); void -lnet_copy_kiov2iov (unsigned int niov, struct kvec *iov, unsigned int iovoffset, - unsigned int nkiov, struct bio_vec *kiov, - unsigned int kiovoffset, - unsigned int nob) +lnet_copy_kiov2iov(unsigned int niov, struct kvec *iov, unsigned int iovoffset, + unsigned int nkiov, struct bio_vec *kiov, + unsigned int kiovoffset, + unsigned int nob) { /* NB iov, kiov are READ-ONLY */ - unsigned int this_nob; - char *addr = NULL; + unsigned int this_nob; + void *kaddr = NULL; + char *addr = NULL; if (nob == 0) return; @@ -480,9 +487,10 @@ lnet_copy_kiov2iov (unsigned int niov, struct kvec *iov, unsigned int iovoffset, (unsigned int)kiov->bv_len - kiovoffset, nob); - if (addr == NULL) - addr = ((char *)kmap(kiov->bv_page)) + - kiov->bv_offset + kiovoffset; + if (addr == NULL) { + kaddr = kmap_local_page(kiov->bv_page); + addr = kaddr + kiov->bv_offset + kiovoffset; + } memcpy((char *)iov->iov_base + iovoffset, addr, this_nob); nob -= this_nob; @@ -499,7 +507,8 @@ lnet_copy_kiov2iov (unsigned int niov, struct kvec *iov, unsigned int iovoffset, addr += this_nob; kiovoffset += this_nob; } else { - kunmap(kiov->bv_page); + kunmap_local(kaddr); + kaddr = NULL; addr = NULL; kiov++; nkiov--; @@ -509,7 +518,7 @@ lnet_copy_kiov2iov (unsigned int niov, struct kvec *iov, unsigned int iovoffset, } while (nob > 0); if (addr != NULL) - kunmap(kiov->bv_page); + kunmap_local(kaddr); } EXPORT_SYMBOL(lnet_copy_kiov2iov); @@ -520,8 +529,9 @@ lnet_copy_iov2kiov(unsigned int nkiov, struct bio_vec *kiov, unsigned int nob) { /* NB kiov, iov are READ-ONLY */ - unsigned int this_nob; - char *addr = NULL; + unsigned int this_nob; + void *kaddr = NULL; + char *addr = NULL; if (nob == 0) return; @@ -551,18 +561,20 @@ lnet_copy_iov2kiov(unsigned int nkiov, struct bio_vec *kiov, (unsigned int)iov->iov_len - iovoffset, nob); - if (addr == NULL) - addr = ((char *)kmap(kiov->bv_page)) + - kiov->bv_offset + kiovoffset; + if (!addr) { + kaddr = kmap_local_page(kiov->bv_page); + addr = kaddr + kiov->bv_offset + kiovoffset; + } - memcpy (addr, (char *)iov->iov_base + iovoffset, this_nob); + memcpy(addr, (char *)iov->iov_base + iovoffset, this_nob); nob -= this_nob; if (kiov->bv_len > kiovoffset + this_nob) { addr += this_nob; kiovoffset += this_nob; } else { - kunmap(kiov->bv_page); + kunmap_local(kaddr); + kaddr = NULL; addr = NULL; kiov++; nkiov--; @@ -578,8 +590,8 @@ lnet_copy_iov2kiov(unsigned int nkiov, struct bio_vec *kiov, } } while (nob > 0); - if (addr != NULL) - kunmap(kiov->bv_page); + if (addr) + kunmap_local(kaddr); } EXPORT_SYMBOL(lnet_copy_iov2kiov); diff --git a/lnet/lnet/lnet-crypto.c b/lnet/lnet/lnet-crypto.c index 26feb7b..0f89c89 100644 --- a/lnet/lnet/lnet-crypto.c +++ b/lnet/lnet/lnet-crypto.c @@ -326,9 +326,9 @@ static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg) goto out_err; } - buf = kmap(page); + buf = kmap_local_page(page); memset(buf, 0xAD, PAGE_SIZE); - kunmap(page); + kunmap_local(buf); for (start = jiffies, end = start + cfs_time_seconds(1) / 4, bcount = 0; time_before(jiffies, end) && err == 0; bcount++) { diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 4b57649..47b2d50 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -879,7 +879,7 @@ struct lu_rdpg { /* for dt_index_walk / mdd_readpage */ void *rdpg_page_get(const struct lu_rdpg *rdpg, unsigned int index); -void rdpg_page_put(const struct lu_rdpg *rdpg, unsigned int index); +void rdpg_page_put(const struct lu_rdpg *rdpg, unsigned int index, void *kaddr); enum lu_xattr_flags { LU_XATTR_REPLACE = BIT(0), diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index aad5eea..31f9306 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -930,8 +930,11 @@ do { \ #endif #ifdef POISON_BULK -#define POISON_PAGE(page, val) do { memset(kmap(page), val, PAGE_SIZE); \ - kunmap(page); } while (0) +#define POISON_PAGE(page, val) do { \ + void *kaddr = kmap_local_page(page, 0); \ + memset(kaddr, val, PAGE_SIZE); \ + kunmap_local(kaddr); \ +} while (0) #else #define POISON_PAGE(page, val) do { } while (0) #endif diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index 01c73a4..4cc44b6 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -163,8 +163,6 @@ struct page *ll_get_dir_page(struct inode *dir, struct md_op_data *op_data, void ll_release_page(struct inode *inode, struct page *page, bool remove) { - kunmap(page); - /* Always remove the page for striped dir, because the page is * built from temporarily in LMV layer */ diff --git a/lustre/llite/file.c b/lustre/llite/file.c index fdd0ebc..96eac18 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -477,12 +477,12 @@ static inline int ll_dom_readpage(void *data, struct page *page) inode = page2inode(page); - kaddr = kmap_atomic(page); + kaddr = ll_kmap_local_page(page); memcpy(kaddr, lnb->lnb_data, lnb->lnb_len); if (lnb->lnb_len < PAGE_SIZE) memset(kaddr + lnb->lnb_len, 0, PAGE_SIZE - lnb->lnb_len); - kunmap_atomic(kaddr); + ll_kunmap_local(kaddr); if (inode && IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) { if (!ll_has_encryption_key(inode)) { @@ -683,10 +683,10 @@ void ll_dir_finish_open(struct inode *inode, struct ptlrpc_request *req) lock_page(page); SetPageUptodate(page); - dp = kmap_atomic(page); + dp = ll_kmap_local_page(page); memcpy(dp, data, PAGE_SIZE); hash = le64_to_cpu(dp->ldp_hash_start); - kunmap_atomic(dp); + ll_kunmap_local(dp); offset = hash_x_index(hash, is_hash64); diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index d5e2f43..91511c3 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -656,10 +656,10 @@ static int ll_prepare_partial_page(const struct lu_env *env, struct cl_io *io, * purposes here we can treat it like i_size. */ if (attr->cat_kms <= offset) { - char *kaddr = kmap_atomic(pg->cp_vmpage); + char *kaddr = ll_kmap_local_page(pg->cp_vmpage); memset(kaddr, 0, PAGE_SIZE); - kunmap_atomic(kaddr); + ll_kunmap_local(kaddr); GOTO(out, result = 0); } diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 475c0e0..cc62e4e 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -2978,7 +2978,10 @@ struct lmv_dir_ctxt { static inline void stripe_dirent_unload(struct stripe_dirent *stripe) { if (stripe->sd_page) { - kunmap(stripe->sd_page); + if (stripe->sd_dp) { + kunmap_local(stripe->sd_dp); + stripe->sd_dp = NULL; + } put_page(stripe->sd_page); stripe->sd_page = NULL; stripe->sd_ent = NULL; @@ -3037,7 +3040,7 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt, LASSERT(!ent); do { - if (stripe->sd_page) { + if (stripe->sd_page && stripe->sd_dp) { __u64 end = le64_to_cpu(stripe->sd_dp->ldp_hash_end); /* @hash should be the last dirent hash */ @@ -3071,6 +3074,7 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt, op_data->op_fid2 = oinfo->lmo_fid; op_data->op_data = oinfo->lmo_root; + stripe->sd_dp = NULL; rc = md_read_page(tgt->ltd_exp, op_data, ctxt->ldc_mrinfo, hash, &stripe->sd_page); @@ -3081,7 +3085,7 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt, if (rc) break; - stripe->sd_dp = page_address(stripe->sd_page); + stripe->sd_dp = kmap_local_page(stripe->sd_page); ent = stripe_dirent_get(ctxt, lu_dirent_start(stripe->sd_dp), stripe_index); /* in case a page filled with ., .. and dummy, read next */ @@ -3228,7 +3232,7 @@ static int lmv_striped_read_page(struct obd_export *exp, RETURN(-ENOMEM); /* Initialize the entry page */ - dp = kmap(page); + dp = kmap_local_page(page); memset(dp, 0, sizeof(*dp)); dp->ldp_hash_start = cpu_to_le64(offset); @@ -3304,6 +3308,7 @@ static int lmv_striped_read_page(struct obd_export *exp, dp->ldp_flags = cpu_to_le32(dp->ldp_flags); dp->ldp_hash_end = cpu_to_le64(ctxt->ldc_hash); + kunmap_local(dp); put_lmv_dir_ctxt(ctxt); OBD_FREE(ctxt, offsetof(typeof(*ctxt), ldc_stripes[stripe_count])); @@ -3312,7 +3317,7 @@ static int lmv_striped_read_page(struct obd_export *exp, RETURN(0); free_page: - kunmap(page); + kunmap_local(dp); __free_page(page); return rc; diff --git a/lustre/lov/lov_page.c b/lustre/lov/lov_page.c index 234e590..86f5aec 100644 --- a/lustre/lov/lov_page.c +++ b/lustre/lov/lov_page.c @@ -217,9 +217,9 @@ int lov_page_init_empty(const struct lu_env *env, struct cl_object *obj, BUILD_BUG_ON(!__same_type(cl_page->cp_lov_index, CP_LOV_INDEX_EMPTY)); cl_page->cp_lov_index = CP_LOV_INDEX_EMPTY; - addr = kmap(cl_page->cp_vmpage); + addr = kmap_local_page(cl_page->cp_vmpage); memset(addr, 0, PAGE_SIZE); - kunmap(cl_page->cp_vmpage); + kunmap_local(addr); SetPageUptodate(cl_page->cp_vmpage); RETURN(0); } diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 42edc22..c131b53 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1144,7 +1144,9 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, */ wait_on_page_locked(page); if (PageUptodate(page)) { - dp = kmap(page); + u32 dflags; + + dp = kmap_local_page(page); if (BITS_PER_LONG == 32 && hash64) { *start = le64_to_cpu(dp->ldp_hash_start) >> 32; *end = le64_to_cpu(dp->ldp_hash_end) >> 32; @@ -1153,6 +1155,8 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, *start = le64_to_cpu(dp->ldp_hash_start); *end = le64_to_cpu(dp->ldp_hash_end); } + dflags = le32_to_cpu(dp->ldp_flags); + kunmap_local(dp); if (unlikely(*start == 1 && *hash == 0)) *hash = *start; else @@ -1163,7 +1167,6 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, "offset %lx [%#llx %#llx], hash %#llx\n", offset, *start, *end, *hash); if (*hash > *end) { - kunmap(page); mdc_release_page(page, 0); page = NULL; } else if (*end != *start && *hash == *end) { @@ -1173,9 +1176,7 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, * mdc_read_page_remote() will issue RPC to * fetch the page we want. */ - kunmap(page); - mdc_release_page(page, - le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE); + mdc_release_page(page, dflags & LDF_COLLIDE); page = NULL; } } else { @@ -1245,7 +1246,8 @@ static void mdc_adjust_dirpages(struct page **pages, int cfs_pgs, int lu_pgs) int i; for (i = 0; i < cfs_pgs; i++) { - struct lu_dirpage *dp = kmap(pages[i]); + void *addr = kmap_local_page(pages[i]); + struct lu_dirpage *dp = addr; struct lu_dirpage *first = dp; struct lu_dirent *end_dirent = NULL; struct lu_dirent *ent; @@ -1286,7 +1288,7 @@ static void mdc_adjust_dirpages(struct page **pages, int cfs_pgs, int lu_pgs) first->ldp_flags &= ~cpu_to_le32(LDF_COLLIDE); first->ldp_flags |= flags & cpu_to_le32(LDF_COLLIDE); - kunmap(pages[i]); + kunmap_local(addr); } LASSERTF(lu_pgs == 0, "left = %d\n", lu_pgs); } @@ -1388,9 +1390,9 @@ static int ll_mdc_read_page_remote(void *data, struct page *page0) SetPageUptodate(page); - dp = kmap(page); + dp = kmap_local_page(page); hash = le64_to_cpu(dp->ldp_hash_start); - kunmap(page); + kunmap_local(dp); offset = hash_x_index(hash, rp->rp_hash64); @@ -1451,6 +1453,7 @@ static int mdc_read_page(struct obd_export *exp, struct md_op_data *op_data, struct lustre_handle lockh; struct ptlrpc_request *enq_req = NULL; struct readpage_param rp_param; + void *addr = NULL; int rc; ENTRY; @@ -1515,7 +1518,6 @@ static int mdc_read_page(struct obd_export *exp, struct md_op_data *op_data, } wait_on_page_locked(page); - (void)kmap(page); if (!PageUptodate(page)) { CERROR("%s: page not updated: "DFID" at %llu: rc %d\n", exp->exp_obd->obd_name, PFID(&op_data->op_fid1), @@ -1530,7 +1532,8 @@ static int mdc_read_page(struct obd_export *exp, struct md_op_data *op_data, } hash_collision: - dp = page_address(page); + addr = kmap_local_page(page); + dp = addr; if (BITS_PER_LONG == 32 && rp_param.rp_hash64) { start = le64_to_cpu(dp->ldp_hash_start) >> 32; end = le64_to_cpu(dp->ldp_hash_end) >> 32; @@ -1554,14 +1557,15 @@ hash_collision: * * XXX not yet. */ + kunmap_local(addr); goto fail; } + kunmap_local(addr); *ppage = page; out_unlock: ldlm_lock_decref(&lockh, it.it_lock_mode); return rc; fail: - kunmap(page); mdc_release_page(page, 1); rc = -EIO; goto out_unlock; diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index fa0815b..d05f222 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -4167,14 +4167,14 @@ int mdd_readpage(const struct lu_env *env, struct md_object *obj, dp->ldp_hash_start = cpu_to_le64(rdpg->rp_hash); dp->ldp_hash_end = cpu_to_le64(MDS_DIR_END_OFF); dp->ldp_flags = cpu_to_le32(LDF_EMPTY); - rdpg_page_put(rdpg, 0); + rdpg_page_put(rdpg, 0, dp); GOTO(out_unlock, rc = LU_PAGE_SIZE); } rc = dt_index_walk(env, mdd_object_child(mdd_obj), rdpg, mdd_dir_page_build, NULL); if (rc >= 0) { - struct lu_dirpage *dp; + struct lu_dirpage *dp; dp = (struct lu_dirpage *)rdpg_page_get(rdpg, 0); dp->ldp_hash_start = cpu_to_le64(rdpg->rp_hash); @@ -4187,7 +4187,7 @@ int mdd_readpage(const struct lu_env *env, struct md_object *obj, dp->ldp_flags = cpu_to_le32(LDF_EMPTY); rc = min_t(unsigned int, LU_PAGE_SIZE, rdpg->rp_count); } - rdpg_page_put(rdpg, 0); + rdpg_page_put(rdpg, 0, dp); } GOTO(out_unlock, rc); diff --git a/lustre/mdt/mdt_io.c b/lustre/mdt/mdt_io.c index 5a9d12b..c47ba19 100644 --- a/lustre/mdt/mdt_io.c +++ b/lustre/mdt/mdt_io.c @@ -2048,7 +2048,7 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, GOTO(buf_put, rc); /* copy data to the buffer finally */ for (i = 0; i < nr_local; i++) { - char *p = kmap(lnb[i].lnb_page); + char *p = kmap_local_page(lnb[i].lnb_page); long off; LASSERT(lnb[i].lnb_page_offset == 0); @@ -2057,7 +2057,7 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, memset(p + off, 0, PAGE_SIZE - off); memcpy(buf + (i << PAGE_SHIFT), p, lnb[i].lnb_len); - kunmap(lnb[i].lnb_page); + kunmap_local(p); copied += lnb[i].lnb_len; } CDEBUG(D_INFO, "Read %i (wanted %u) bytes from %llu\n", copied, diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index f1b61d2..74ec530 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -1602,11 +1602,11 @@ again: int rc2; union lu_page *ptr; - ptr = kmap(pages[i]); + ptr = kmap_local_page(pages[i]); rc2 = mgc_apply_recover_logs(obd, cld, res->mcr_offset, ptr, min_t(int, ealen, PAGE_SIZE), mne_swab); - kunmap(pages[i]); + kunmap_local(ptr); if (rc2 < 0) { CWARN("%s: error processing %s log recovery: rc = %d\n", obd->obd_name, diff --git a/lustre/mgc/mgc_request_server.c b/lustre/mgc/mgc_request_server.c index 91e2803..b375621 100644 --- a/lustre/mgc/mgc_request_server.c +++ b/lustre/mgc/mgc_request_server.c @@ -480,10 +480,10 @@ again: union lu_page *ptr; int rc2; - ptr = kmap(pages[i]); + ptr = kmap_local_page(pages[i]); rc2 = nodemap_process_idx_pages(new_config, ptr, &recent_nodemap); - kunmap(pages[i]); + kunmap_local(ptr); if (rc2 < 0) { CWARN("%s: error processing %s log nodemap: rc = %d\n", obd->obd_name, diff --git a/lustre/mgs/mgs_nids.c b/lustre/mgs/mgs_nids.c index 5f1722b..42dce3b 100644 --- a/lustre/mgs/mgs_nids.c +++ b/lustre/mgs/mgs_nids.c @@ -64,6 +64,7 @@ static int mgs_nidtbl_read(struct obd_export *exp, struct mgs_nidtbl *tbl, __u64 version = res->mcr_offset; bool nobuf = false; void *buf = NULL; + void *vaddr = NULL; int bytes_in_unit = 0; int units_in_page = 0; int index = 0; @@ -135,10 +136,10 @@ static int mgs_nidtbl_read(struct obd_export *exp, struct mgs_nidtbl *tbl, /* destroy previous map */ if (index > 0) - kunmap(pages[index - 1]); + kunmap_local(vaddr); /* reassign buffer */ - buf = kmap(pages[index]); + buf = vaddr = kmap_local_page(pages[index]); ++index; units_in_page = PAGE_SIZE / unit_size; @@ -225,7 +226,7 @@ static int mgs_nidtbl_read(struct obd_export *exp, struct mgs_nidtbl *tbl, bytes_in_unit, index, nrpages, units_total); } if (index > 0) - kunmap(pages[index - 1]); + kunmap_local(vaddr); out: LASSERT(version <= tbl->mn_version); res->mcr_size = tbl->mn_version; diff --git a/lustre/obdclass/dt_object.c b/lustre/obdclass/dt_object.c index 10956de..599aeca 100644 --- a/lustre/obdclass/dt_object.c +++ b/lustre/obdclass/dt_object.c @@ -799,18 +799,18 @@ void *rdpg_page_get(const struct lu_rdpg *rdpg, unsigned int index) { if (rdpg->rp_npages) { LASSERT(index < rdpg->rp_npages); - return kmap(rdpg->rp_pages[index]); + return kmap_local_page(rdpg->rp_pages[index]); } - LASSERT(index * PAGE_SIZE < rdpg->rp_count); + LASSERT(index << PAGE_SHIFT < rdpg->rp_count); - return rdpg->rp_data + index * PAGE_SIZE; + return rdpg->rp_data + (index << PAGE_SHIFT); } EXPORT_SYMBOL(rdpg_page_get); -void rdpg_page_put(const struct lu_rdpg *rdpg, unsigned int index) +void rdpg_page_put(const struct lu_rdpg *rdpg, unsigned int index, void *kaddr) { if (rdpg->rp_npages) - kunmap(rdpg->rp_pages[index]); + kunmap_local(kaddr); } EXPORT_SYMBOL(rdpg_page_put); @@ -886,10 +886,11 @@ int dt_index_walk(const struct lu_env *env, struct dt_object *obj, * rc < 0 -> error. */ for (pageidx = 0; rc == 0 && bytes > 0; pageidx++) { + void *addr; union lu_page *lp; int i; - lp = rdpg_page_get(rdpg, pageidx); + lp = addr = rdpg_page_get(rdpg, pageidx); /* fill lu pages */ for (i = 0; i < LU_PAGE_COUNT; i++, lp++, bytes-=LU_PAGE_SIZE) { rc = filler(env, obj, lp, @@ -903,7 +904,7 @@ int dt_index_walk(const struct lu_env *env, struct dt_object *obj, /* end of index */ break; } - rdpg_page_put(rdpg, pageidx); + rdpg_page_put(rdpg, pageidx, addr); } out: @@ -1034,9 +1035,11 @@ void dt_index_page_adjust(struct page **pages, const u32 npages, int i; if (nlupgs_mod) { + void *kaddr; + pgidx = nlupgs / LU_PAGE_COUNT; LASSERT(pgidx < npages); - lp = kmap(pages[pgidx]); + lp = kaddr = kmap_local_page(pages[pgidx]); remain_nlupgs = LU_PAGE_COUNT - nlupgs_mod; /* initialize the header for the remain lu_pages */ @@ -1046,7 +1049,7 @@ void dt_index_page_adjust(struct page **pages, const u32 npages, lip->lip_magic = LIP_MAGIC; } - kunmap(pages[pgidx]); + kunmap_local(kaddr); } } #else diff --git a/lustre/obdclass/integrity.c b/lustre/obdclass/integrity.c index 9034da5..69fc348 100644 --- a/lustre/obdclass/integrity.c +++ b/lustre/obdclass/integrity.c @@ -35,13 +35,15 @@ int obd_page_dif_generate_buffer(const char *obd_name, struct page *page, { unsigned int off = start; unsigned int end = start + length; + void *addr; char *data_buf; __be16 *guard_buf = guard_start; unsigned int data_size; int guard_used = 0; int rc = 0; - data_buf = kmap(page) + start; + addr = kmap_local_page(page); + data_buf = addr + start; while (off < end) { if (guard_used >= guard_number) { rc = -E2BIG; @@ -59,7 +61,7 @@ int obd_page_dif_generate_buffer(const char *obd_name, struct page *page, } *used_number = guard_used; out: - kunmap(page); + kunmap_local(addr); return rc; } @@ -102,7 +104,7 @@ static int __obd_t10_performance_test(const char *obd_name, GOTO(out, rc); } - buffer = kmap(__page); + buffer = kmap_local_page(__page); guard_start = (__be16 *)buffer; guard_number = PAGE_SIZE / sizeof(*guard_start); for (i = 0; i < repeat_number; i++) { @@ -124,7 +126,7 @@ static int __obd_t10_performance_test(const char *obd_name, used_number = 0; } } - kunmap(__page); + kunmap_local(buffer); if (rc) GOTO(out_final, rc); @@ -205,9 +207,9 @@ static void obd_t10_performance_test(const char *obd_name, goto out; } - buf = kmap(page); + buf = kmap_local_page(page); memset(buf, 0xAD, PAGE_SIZE); - kunmap(page); + kunmap_local(buf); for (start = jiffies, end = start + cfs_time_seconds(1) / 4, bcount = 0; time_before(jiffies, end) && rc == 0; bcount++) { diff --git a/lustre/obdclass/jobid.c b/lustre/obdclass/jobid.c index 07289e7..cd5c4bb 100644 --- a/lustre/obdclass/jobid.c +++ b/lustre/obdclass/jobid.c @@ -246,7 +246,7 @@ static int cfs_access_process_vm(struct task_struct *tsk, if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; - maddr = kmap(page); + maddr = kmap_local_page(page); if (write) { copy_to_user_page(vma, page, addr, maddr + offset, buf, bytes); @@ -255,7 +255,7 @@ static int cfs_access_process_vm(struct task_struct *tsk, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - kunmap(page); + kunmap_local(maddr); put_page(page); len -= bytes; buf += bytes; diff --git a/lustre/obdecho/echo.c b/lustre/obdecho/echo.c index 13252f9..c3b8fab 100644 --- a/lustre/obdecho/echo.c +++ b/lustre/obdecho/echo.c @@ -140,8 +140,9 @@ static void echo_page_debug_setup(struct page *page, int rw, u64 id, __u64 offset, int len) { - int page_offset = offset & ~PAGE_MASK; - char *addr = ((char *)kmap(page)) + page_offset; + int page_offset = offset & ~PAGE_MASK; + char *kaddr = kmap_local_page(page); + char *addr = kaddr + page_offset; if (len % OBD_ECHO_BLOCK_SIZE != 0) CERROR("Unexpected block size %d\n", len); @@ -160,17 +161,18 @@ echo_page_debug_setup(struct page *page, int rw, u64 id, len -= OBD_ECHO_BLOCK_SIZE; } - kunmap(page); + kunmap_local(kaddr); } static int echo_page_debug_check(struct page *page, u64 id, __u64 offset, int len) { - int page_offset = offset & ~PAGE_MASK; - char *addr = ((char *)kmap(page)) + page_offset; - int rc = 0; - int rc2; + int page_offset = offset & ~PAGE_MASK; + char *kaddr = kmap_local_page(page); + char *addr = kaddr + page_offset; + int rc = 0; + int rc2; if (len % OBD_ECHO_BLOCK_SIZE != 0) CERROR("Unexpected block size %d\n", len); @@ -187,7 +189,7 @@ echo_page_debug_check(struct page *page, u64 id, len -= OBD_ECHO_BLOCK_SIZE; } - kunmap(page); + kunmap_local(kaddr); return rc; } @@ -277,7 +279,7 @@ static int echo_finalize_lb(struct obdo *oa, struct obd_ioobj *obj, for (i = 0; i < count; i++, (*pgs) ++, res++) { struct page *page = res->lnb_page; - void *addr; + void *addr; if (!page) { CERROR("null page objid %llu:%p, buf %d/%d\n", @@ -286,7 +288,7 @@ static int echo_finalize_lb(struct obdo *oa, struct obd_ioobj *obj, return -EFAULT; } - addr = kmap(page); + addr = kmap_local_page(page); CDEBUG(D_PAGE, "$$$$ use page %p, addr %p@%llu\n", res->lnb_page, addr, res->lnb_file_offset); @@ -301,7 +303,7 @@ static int echo_finalize_lb(struct obdo *oa, struct obd_ioobj *obj, rc = vrc; } - kunmap(page); + kunmap_local(addr); /* NB see comment above regarding persistent pages */ __free_page(page); } @@ -373,7 +375,6 @@ preprw_cleanup: */ CERROR("cleaning up %u pages (%d obdos)\n", *pages, objcount); for (i = 0; i < *pages; i++) { - kunmap(res[i].lnb_page); /* * NB if this is a persistent page, __free_page() will just * lose the extra ref gained above @@ -954,7 +955,8 @@ void echo_persistent_pages_fini(void) int echo_persistent_pages_init(void) { struct page *pg; - int i; + void *kaddr; + int i; for (i = 0; i < ECHO_PERSISTENT_PAGES; i++) { gfp_t gfp_mask = (i < ECHO_PERSISTENT_PAGES / 2) ? @@ -966,8 +968,9 @@ int echo_persistent_pages_init(void) return -ENOMEM; } - memset(kmap(pg), 0, PAGE_SIZE); - kunmap(pg); + kaddr = kmap_local_page(pg); + memset(kaddr, 0, PAGE_SIZE); + kunmap_local(kaddr); /* set mapping so page is not considered encrypted */ pg->mapping = ECHO_MAPPING_UNENCRYPTED; diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 801ab94..2ad9300 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -2045,7 +2045,7 @@ static void echo_client_page_debug_setup(struct page *page, int rw, u64 id, /* no partial pages on the client */ LASSERT(count == PAGE_SIZE); - addr = kmap(page); + addr = kmap_local_page(page); for (delta = 0; delta < PAGE_SIZE; delta += OBD_ECHO_BLOCK_SIZE) { if (rw == OBD_BRW_WRITE) { @@ -2059,7 +2059,7 @@ static void echo_client_page_debug_setup(struct page *page, int rw, u64 id, stripe_off, stripe_id); } - kunmap(page); + kunmap_local(addr); } static int @@ -2075,7 +2075,7 @@ echo_client_page_debug_check(struct page *page, u64 id, u64 offset, u64 count) /* no partial pages on the client */ LASSERT(count == PAGE_SIZE); - addr = kmap(page); + addr = kmap_local_page(page); for (rc = delta = 0; delta < PAGE_SIZE; delta += OBD_ECHO_BLOCK_SIZE) { stripe_off = offset + delta; @@ -2090,7 +2090,7 @@ echo_client_page_debug_check(struct page *page, u64 id, u64 offset, u64 count) } } - kunmap(page); + kunmap_local(addr); return rc; } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index cf5e0a7..123da66 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1058,6 +1058,7 @@ EXPORT_SYMBOL(osc_init_grant); static void handle_short_read(int nob_read, size_t page_count, struct brw_page **pga) { + void *kaddr; char *ptr; int i = 0; @@ -1066,11 +1067,11 @@ static void handle_short_read(int nob_read, size_t page_count, LASSERT(page_count > 0); if (pga[i]->bp_count > nob_read) { + kaddr = kmap_local_page(pga[i]->bp_page); /* EOF inside this page */ - ptr = kmap(pga[i]->bp_page) + - (pga[i]->bp_off & ~PAGE_MASK); + ptr = kaddr + (pga[i]->bp_off & ~PAGE_MASK); memset(ptr + nob_read, 0, pga[i]->bp_count - nob_read); - kunmap(pga[i]->bp_page); + kunmap_local(kaddr); page_count--; i++; break; @@ -1083,9 +1084,10 @@ static void handle_short_read(int nob_read, size_t page_count, /* zero remaining pages */ while (page_count-- > 0) { - ptr = kmap(pga[i]->bp_page) + (pga[i]->bp_off & ~PAGE_MASK); + kaddr = kmap_local_page(pga[i]->bp_page); + ptr = kaddr + (pga[i]->bp_off & ~PAGE_MASK); memset(ptr, 0, pga[i]->bp_count); - kunmap(pga[i]->bp_page); + kunmap_local(kaddr); i++; } } @@ -1185,7 +1187,7 @@ static int osc_checksum_bulk_t10pi(const char *obd_name, int nob, GOTO(out, rc); } - buffer = kmap(__page); + buffer = kmap_local_page(__page); guard_start = (__be16 *)buffer; guard_number = PAGE_SIZE / sizeof(*guard_start); CDEBUG(D_PAGE | (resend ? D_HA : 0), @@ -1210,10 +1212,10 @@ static int osc_checksum_bulk_t10pi(const char *obd_name, int nob, */ if (unlikely(i == 0 && opc == OST_READ && CFS_FAIL_CHECK(OBD_FAIL_OSC_CHECKSUM_RECEIVE))) { - unsigned char *ptr = kmap(pga[i]->bp_page); + void *ptr = kmap_local_page(pga[i]->bp_page); memcpy(ptr + off, "bad1", min_t(typeof(nob), 4, nob)); - kunmap(pga[i]->bp_page); + kunmap_local(ptr); } /* @@ -1241,7 +1243,7 @@ static int osc_checksum_bulk_t10pi(const char *obd_name, int nob, pg_count--; i++; } - kunmap(__page); + kunmap_local(buffer); if (rc) GOTO(out_hash, rc); @@ -1309,11 +1311,11 @@ static int osc_checksum_bulk(int nob, size_t pg_count, */ if (i == 0 && opc == OST_READ && CFS_FAIL_CHECK(OBD_FAIL_OSC_CHECKSUM_RECEIVE)) { - unsigned char *ptr = kmap(pga[i]->bp_page); + void *ptr = kmap_local_page(pga[i]->bp_page); int off = pga[i]->bp_off & ~PAGE_MASK; memcpy(ptr + off, "bad1", min_t(typeof(nob), 4, nob)); - kunmap(pga[i]->bp_page); + kunmap_local(ptr); } cfs_crypto_hash_update_page(req, pga[i]->bp_page, pga[i]->bp_off & ~PAGE_MASK, @@ -1854,13 +1856,13 @@ no_bulk: LASSERT((pga[0]->bp_flag & OBD_BRW_SRVLOCK) == (pg->bp_flag & OBD_BRW_SRVLOCK)); if (short_io_size != 0 && opc == OST_WRITE) { - unsigned char *ptr = kmap_atomic(pg->bp_page); + unsigned char *ptr = ll_kmap_local_page(pg->bp_page); LASSERT(short_io_size >= requested_nob + pg->bp_count); memcpy(short_io_buf + requested_nob, ptr + poff, pg->bp_count); - kunmap_atomic(ptr); + ll_kunmap_local(ptr); } else if (short_io_size == 0) { desc->bd_frag_ops->add_kiov_frag(desc, pg->bp_page, poff, pg->bp_count); @@ -2045,7 +2047,7 @@ static void dump_all_bulk_pages(struct obdo *oa, __u32 page_count, for (i = 0; i < page_count; i++) { len = pga[i]->bp_count; - buf = kmap(pga[i]->bp_page); + buf = kmap_local_page(pga[i]->bp_page); while (len != 0) { rc = cfs_kernel_write(filp, buf, len, &filp->f_pos); if (rc < 0) { @@ -2056,7 +2058,7 @@ static void dump_all_bulk_pages(struct obdo *oa, __u32 page_count, len -= rc; buf += rc; } - kunmap(pga[i]->bp_page); + kunmap_local(buf); } rc = vfs_fsync_range(filp, 0, LLONG_MAX, 1); @@ -2263,10 +2265,10 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc) CDEBUG(D_CACHE, "page %p count %d\n", aa->aa_ppga[i]->bp_page, count); - ptr = kmap_atomic(aa->aa_ppga[i]->bp_page); + ptr = ll_kmap_local_page(aa->aa_ppga[i]->bp_page); memcpy(ptr + (aa->aa_ppga[i]->bp_off & ~PAGE_MASK), buf, count); - kunmap_atomic((void *) ptr); + ll_kunmap_local((void *) ptr); buf += count; nob -= count; diff --git a/lustre/osd-ldiskfs/osd_integrity.c b/lustre/osd-ldiskfs/osd_integrity.c index d157d58..7589c78 100644 --- a/lustre/osd-ldiskfs/osd_integrity.c +++ b/lustre/osd-ldiskfs/osd_integrity.c @@ -261,10 +261,10 @@ static void bio_integrity_fault_inject(struct bio *bio) bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; - kaddr = kmap(page); + kaddr = kmap_local_page(page); addr = kaddr; *addr = ~(*addr); - kunmap(page); + kunmap_local(kaddr); break; } } diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index 284266a..5ca6e91 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -421,8 +421,10 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, nblocks = 1; if (blocks[block_idx + i] == 0) { /* hole */ + void *addr; struct niobuf_local *lnb = iobuf->dr_lnbs[page_idx]; + CDEBUG(D_INODE, "hole at page_idx %d, block_idx %d, at offset %llu\n", page_idx, block_idx, @@ -434,8 +436,9 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, page_idx, block_idx, i, (unsigned long long)start_blocks, (unsigned long long)count, npages); - memset(kmap(page) + page_offset, 0, blocksize); - kunmap(page); + addr = kmap_local_page(page); + memset(addr + page_offset, 0, blocksize); + kunmap_local(addr); continue; } @@ -1161,7 +1164,7 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, osd_iobuf_add_page(iobuf, &lnb[i]); } else { long off; - char *p = kmap(lnb[i].lnb_page); + char *p = kmap_local_page(lnb[i].lnb_page); off = lnb[i].lnb_page_offset; if (off) @@ -1170,7 +1173,7 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, ~PAGE_MASK; if (off) memset(p + off, 0, PAGE_SIZE - off); - kunmap(lnb[i].lnb_page); + kunmap_local(p); } } end = ktime_get(); diff --git a/lustre/osd-zfs/osd_io.c b/lustre/osd-zfs/osd_io.c index 5e4cffe..9a105af 100644 --- a/lustre/osd-zfs/osd_io.c +++ b/lustre/osd-zfs/osd_io.c @@ -1040,10 +1040,12 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, continue; if (lnb[i].lnb_page->mapping == (void *)obj) { + void *addr = kmap_local_page(lnb[i].lnb_page); + osd_dmu_write(osd, obj->oo_dn, lnb[i].lnb_file_offset, - lnb[i].lnb_len, kmap(lnb[i].lnb_page) + + lnb[i].lnb_len, addr + lnb[i].lnb_page_offset, oh->ot_tx); - kunmap(lnb[i].lnb_page); + kunmap_local(addr); iosize += lnb[i].lnb_len; abufsz = lnb[i].lnb_len; /* to drop cache below */ } else if (lnb[i].lnb_data) { diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 12fa688..b7885ad 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -358,7 +358,7 @@ struct osp_it { __u64 ooi_next; struct dt_object *ooi_obj; void *ooi_ent; - struct page *ooi_cur_page; + void *ooi_cur_kaddr; struct lu_idxpage *ooi_cur_idxpage; struct page **ooi_pages; }; diff --git a/lustre/osp/osp_object.c b/lustre/osp/osp_object.c index 7e4b0bd..3a1cc73 100644 --- a/lustre/osp/osp_object.c +++ b/lustre/osp/osp_object.c @@ -1756,10 +1756,10 @@ void osp_it_fini(const struct lu_env *env, struct dt_it *di) if (pages != NULL) { for (i = 0; i < npages; i++) { - if (pages[i] != NULL) { - if (pages[i] == it->ooi_cur_page) { - kunmap(pages[i]); - it->ooi_cur_page = NULL; + if (pages[i]) { + if (it->ooi_cur_kaddr) { + kunmap_local(it->ooi_cur_kaddr); + it->ooi_cur_kaddr = NULL; } __free_page(pages[i]); } @@ -1937,7 +1937,7 @@ finish_cur_idxpage: process_page: if (it->ooi_pos_lu_page < LU_PAGE_COUNT) { - it->ooi_cur_idxpage = (void *)it->ooi_cur_page + + it->ooi_cur_idxpage = (void *)it->ooi_cur_kaddr + LU_PAGE_SIZE * it->ooi_pos_lu_page; if (it->ooi_swab) lustre_swab_lip_header(it->ooi_cur_idxpage); @@ -1959,14 +1959,15 @@ process_page: goto process_idxpage; } - kunmap(it->ooi_cur_page); - it->ooi_cur_page = NULL; + kunmap_local(it->ooi_cur_kaddr); + it->ooi_cur_kaddr = NULL; it->ooi_pos_page++; start: pages = it->ooi_pages; if (it->ooi_pos_page < it->ooi_valid_npages) { - it->ooi_cur_page = kmap(pages[it->ooi_pos_page]); + it->ooi_cur_kaddr = + kmap_local_page(pages[it->ooi_pos_page]); it->ooi_pos_lu_page = 0; goto process_page; } @@ -1982,7 +1983,7 @@ start: it->ooi_valid_npages = 0; it->ooi_swab = 0; it->ooi_ent = NULL; - it->ooi_cur_page = NULL; + it->ooi_cur_kaddr = NULL; it->ooi_cur_idxpage = NULL; it->ooi_pages = NULL; } diff --git a/lustre/ptlrpc/sec_plain.c b/lustre/ptlrpc/sec_plain.c index 624a458..cb9b58f 100644 --- a/lustre/ptlrpc/sec_plain.c +++ b/lustre/ptlrpc/sec_plain.c @@ -138,10 +138,10 @@ static void corrupt_bulk_data(struct ptlrpc_bulk_desc *desc) if (desc->bd_vec[i].bv_len == 0) continue; - ptr = kmap(desc->bd_vec[i].bv_page); + ptr = kmap_local_page(desc->bd_vec[i].bv_page); off = desc->bd_vec[i].bv_offset & ~PAGE_MASK; ptr[off] ^= 0x1; - kunmap(desc->bd_vec[i].bv_page); + kunmap_local(ptr); return; } } diff --git a/lustre/quota/qsd_reint.c b/lustre/quota/qsd_reint.c index 63781af..428a294 100644 --- a/lustre/quota/qsd_reint.c +++ b/lustre/quota/qsd_reint.c @@ -111,7 +111,8 @@ static int qsd_reint_entries(const struct lu_env *env, size = ii->ii_recsize + ii->ii_keysize; for (i = 0; i < npages; i++) { - union lu_page *lip = kmap(pages[i]); + void *kaddr = kmap_local_page(pages[i]); + union lu_page *lip = kaddr; for (j = 0; j < LU_PAGE_COUNT; j++) { if (need_swab) @@ -161,7 +162,7 @@ static int qsd_reint_entries(const struct lu_env *env, lip++; } out: - kunmap(pages[i]); + kunmap_local(kaddr); if (rc) break; } diff --git a/lustre/target/tgt_handler.c b/lustre/target/tgt_handler.c index 747dbd1..75fe1f7 100644 --- a/lustre/target/tgt_handler.c +++ b/lustre/target/tgt_handler.c @@ -1847,12 +1847,13 @@ static int tgt_checksum_niobuf(struct lu_target *tgt, struct page *np = tgt_page_to_corrupt; if (np) { - char *ptr = kmap_atomic(local_nb[i].lnb_page); - char *ptr2 = page_address(np); + char *ptr = ll_kmap_local_page(local_nb[i].lnb_page); + char *ptr2 = ll_kmap_local_page(np); memcpy(ptr2 + off, ptr + off, len); memcpy(ptr2 + off, "bad3", min(4, len)); - kunmap_atomic(ptr); + ll_kunmap_local(ptr2); + ll_kunmap_local(ptr); /* LU-8376 to preserve original index for * display in dump_all_bulk_pages() */ @@ -1879,12 +1880,13 @@ static int tgt_checksum_niobuf(struct lu_target *tgt, struct page *np = tgt_page_to_corrupt; if (np) { - char *ptr = kmap_atomic(local_nb[i].lnb_page); - char *ptr2 = page_address(np); + char *ptr = ll_kmap_local_page(local_nb[i].lnb_page); + char *ptr2 = ll_kmap_local_page(np); memcpy(ptr2 + off, ptr + off, len); memcpy(ptr2 + off, "bad4", min(4, len)); - kunmap_atomic(ptr); + ll_kunmap_local(ptr2); + ll_kunmap_local(ptr); /* LU-8376 to preserve original index for * display in dump_all_bulk_pages() */ @@ -1949,8 +1951,10 @@ static void dump_all_bulk_pages(struct obdo *oa, int count, } for (i = 0; i < count; i++) { + void *addr = kmap_local_page(local_nb[i].lnb_page); + len = local_nb[i].lnb_len; - buf = kmap(local_nb[i].lnb_page); + buf = addr; while (len != 0) { rc = cfs_kernel_write(filp, buf, len, &filp->f_pos); if (rc < 0) { @@ -1961,7 +1965,7 @@ static void dump_all_bulk_pages(struct obdo *oa, int count, len -= rc; buf += rc; } - kunmap(local_nb[i].lnb_page); + kunmap_local(addr); } rc = vfs_fsync_range(filp, 0, LLONG_MAX, 1); @@ -2038,9 +2042,9 @@ static int tgt_pages2shortio(struct niobuf_local *local, int npages, if (len > size) return -EINVAL; - ptr = kmap_atomic(local[i].lnb_page); + ptr = ll_kmap_local_page(local[i].lnb_page); memcpy(buf, ptr + off, len); - kunmap_atomic(ptr); + ll_kunmap_local(ptr); buf += len; size -= len; } @@ -2081,7 +2085,7 @@ static int tgt_checksum_niobuf_t10pi(struct lu_target *tgt, goto out; } - buffer = kmap(__page); + buffer = kmap_local_page(__page); guard_start = (__be16 *)buffer; guard_number = PAGE_SIZE / sizeof(*guard_start); if (unlikely(resend)) @@ -2106,12 +2110,13 @@ static int tgt_checksum_niobuf_t10pi(struct lu_target *tgt, struct page *np = tgt_page_to_corrupt; if (np) { - char *ptr = kmap_atomic(local_nb[i].lnb_page); - char *ptr2 = page_address(np); + char *ptr = ll_kmap_local_page(local_nb[i].lnb_page); + char *ptr2 = ll_kmap_local_page(np); memcpy(ptr2 + off, ptr + off, len); memcpy(ptr2 + off, "bad3", min(4, len)); - kunmap_atomic(ptr); + ll_kunmap_local(ptr2); + ll_kunmap_local(ptr); /* LU-8376 to preserve original index for * display in dump_all_bulk_pages() */ @@ -2226,12 +2231,13 @@ static int tgt_checksum_niobuf_t10pi(struct lu_target *tgt, struct page *np = tgt_page_to_corrupt; if (np) { - char *ptr = kmap_atomic(local_nb[i].lnb_page); - char *ptr2 = page_address(np); + char *ptr = ll_kmap_local_page(local_nb[i].lnb_page); + char *ptr2 = ll_kmap_local_page(np); memcpy(ptr2 + off, ptr + off, len); memcpy(ptr2 + off, "bad4", min(4, len)); - kunmap_atomic(ptr); + ll_kunmap_local(ptr2); + ll_kunmap_local(ptr); /* LU-8376 to preserve original index for * display in dump_all_bulk_pages() */ @@ -2246,7 +2252,7 @@ static int tgt_checksum_niobuf_t10pi(struct lu_target *tgt, } } } - kunmap(__page); + kunmap_local(buffer); if (rc) GOTO(out_hash, rc); @@ -2580,11 +2586,9 @@ static int tgt_shortio2pages(struct niobuf_local *local, int npages, CDEBUG(D_PAGE, "index %d offset = %d len = %d left = %d\n", i, off, len, size); - ptr = kmap_atomic(local[i].lnb_page); - if (ptr == NULL) - return -EINVAL; + ptr = ll_kmap_local_page(local[i].lnb_page); memcpy(ptr + off, buf, len < size ? len : size); - kunmap_atomic(ptr); + ll_kunmap_local(ptr); buf += len; size -= len; } -- 1.8.3.1