X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Flib-md.c;h=ced8a775a17bf7bdc4a9b91f5c4973b7a91cf0d2;hp=c78221cceac34ef7c2473e1edc4b87f3708f19d8;hb=b7035222bd649d66beaa8dc8774ff53623fa54dd;hpb=2104ed0f0da3651f0cb4ab0c78a1037891d7cb4f;ds=sidebyside diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index c78221c..ced8a77 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2012, 2013, Intel Corporation. + * Copyright (c) 2012, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -44,35 +40,35 @@ /* must be called with lnet_res_lock held */ void -lnet_md_unlink(lnet_libmd_t *md) +lnet_md_unlink(struct lnet_libmd *md) { - if ((md->md_flags & LNET_MD_FLAG_ZOMBIE) == 0) { - /* first unlink attempt... */ - lnet_me_t *me = md->md_me; + if ((md->md_flags & LNET_MD_FLAG_ZOMBIE) == 0) { + /* first unlink attempt... */ + struct lnet_me *me = md->md_me; - md->md_flags |= LNET_MD_FLAG_ZOMBIE; + md->md_flags |= LNET_MD_FLAG_ZOMBIE; - /* Disassociate from ME (if any), and unlink it if it was created - * with LNET_UNLINK */ - if (me != NULL) { + /* Disassociate from ME (if any), and unlink it if it was created + * with LNET_UNLINK */ + if (me != NULL) { /* detach MD from portal */ lnet_ptl_detach_md(me, md); - if (me->me_unlink == LNET_UNLINK) - lnet_me_unlink(me); - } + if (me->me_unlink == LNET_UNLINK) + lnet_me_unlink(me); + } - /* ensure all future handle lookups fail */ + /* ensure all future handle lookups fail */ lnet_res_lh_invalidate(&md->md_lh); - } + } - if (md->md_refcount != 0) { - CDEBUG(D_NET, "Queueing unlink of md %p\n", md); - return; - } + if (md->md_refcount != 0) { + CDEBUG(D_NET, "Queueing unlink of md %p\n", md); + return; + } - CDEBUG(D_NET, "Unlinking md %p\n", md); + CDEBUG(D_NET, "Unlinking md %p\n", md); - if (md->md_eq != NULL) { + if (md->md_eq != NULL) { int cpt = lnet_cpt_of_cookie(md->md_lh.lh_cookie); LASSERT(*md->md_eq->eq_refs[cpt] > 0); @@ -84,86 +80,196 @@ lnet_md_unlink(lnet_libmd_t *md) lnet_md_free(md); } -static int -lnet_md_build(lnet_libmd_t *lmd, lnet_md_t *umd, int unlink) +struct page * +lnet_kvaddr_to_page(unsigned long vaddr) { - int i; - unsigned int niov; - int total_length = 0; - - lmd->md_me = NULL; - lmd->md_start = umd->start; - lmd->md_offset = 0; - lmd->md_max_size = umd->max_size; - lmd->md_options = umd->options; - lmd->md_user_ptr = umd->user_ptr; - lmd->md_eq = NULL; - lmd->md_threshold = umd->threshold; - lmd->md_refcount = 0; - lmd->md_flags = (unlink == LNET_UNLINK) ? LNET_MD_FLAG_AUTO_UNLINK : 0; + if (is_vmalloc_addr((void *)vaddr)) + return vmalloc_to_page((void *)vaddr); - if ((umd->options & LNET_MD_IOVEC) != 0) { +#ifdef CONFIG_HIGHMEM - if ((umd->options & LNET_MD_KIOV) != 0) /* Can't specify both */ - return -EINVAL; +#ifdef HAVE_KMAP_TO_PAGE + /* + * This ifdef is added to handle the kernel versions + * which have kmap_to_page() function exported. If so, + * we should use it. Otherwise, remain with the legacy check. + */ + return kmap_to_page((void *)vaddr); +#else - lmd->md_niov = niov = umd->length; - memcpy(lmd->md_iov.iov, umd->start, - niov * sizeof (lmd->md_iov.iov[0])); + if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) { + /* No highmem pages only used for bulk (kiov) I/O */ + CERROR("find page for address in highmem\n"); + LBUG(); + } + return virt_to_page(vaddr); +#endif /* HAVE_KMAP_TO_PAGE */ +#else - for (i = 0; i < (int)niov; i++) { - /* We take the base address on trust */ - if (lmd->md_iov.iov[i].iov_len <= 0) /* invalid length */ - return -EINVAL; + return virt_to_page(vaddr); +#endif /* CONFIG_HIGHMEM */ +} +EXPORT_SYMBOL(lnet_kvaddr_to_page); - total_length += lmd->md_iov.iov[i].iov_len; - } +int +lnet_cpt_of_md(struct lnet_libmd *md, unsigned int offset) +{ + int cpt = CFS_CPT_ANY; + unsigned int niov; - lmd->md_length = total_length; + /* + * if the md_options has a bulk handle then we want to look at the + * bulk md because that's the data which we will be DMAing + */ + if (md && (md->md_options & LNET_MD_BULK_HANDLE) != 0 && + !LNetMDHandleIsInvalid(md->md_bulk_handle)) + md = lnet_handle2md(&md->md_bulk_handle); + + if (!md || md->md_niov == 0) + return CFS_CPT_ANY; + + niov = md->md_niov; + + /* + * There are three cases to handle: + * 1. The MD is using lnet_kiov_t + * 2. The MD is using struct kvec + * 3. Contiguous buffer allocated via vmalloc + * + * in case 2 we can use virt_to_page() macro to get the page + * address of the memory kvec describes. + * + * in case 3 use is_vmalloc_addr() and vmalloc_to_page() + * + * The offset provided can be within the first iov/kiov entry or + * it could go beyond it. In that case we need to make sure to + * look at the page which actually contains the data that will be + * DMAed. + */ + if ((md->md_options & LNET_MD_KIOV) != 0) { + lnet_kiov_t *kiov = md->md_iov.kiov; + + while (offset >= kiov->kiov_len) { + offset -= kiov->kiov_len; + niov--; + kiov++; + if (niov == 0) { + CERROR("offset %d goes beyond kiov\n", offset); + goto out; + } + } + + cpt = cfs_cpt_of_node(lnet_cpt_table(), + page_to_nid(kiov->kiov_page)); + } else { + struct kvec *iov = md->md_iov.iov; + unsigned long vaddr; + struct page *page; + + while (offset >= iov->iov_len) { + offset -= iov->iov_len; + niov--; + iov++; + if (niov == 0) { + CERROR("offset %d goes beyond iov\n", offset); + goto out; + } + } + + vaddr = ((unsigned long)iov->iov_base) + offset; + page = lnet_kvaddr_to_page(vaddr); + if (!page) { + CERROR("Couldn't resolve vaddr 0x%lx to page\n", vaddr); + goto out; + } + cpt = cfs_cpt_of_node(lnet_cpt_table(), page_to_nid(page)); + } - if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ - (umd->max_size < 0 || - umd->max_size > total_length)) // illegal max_size - return -EINVAL; +out: + return cpt; +} - } else if ((umd->options & LNET_MD_KIOV) != 0) { - lmd->md_niov = niov = umd->length; - memcpy(lmd->md_iov.kiov, umd->start, - niov * sizeof (lmd->md_iov.kiov[0])); +static int +lnet_md_build(struct lnet_libmd *lmd, struct lnet_md *umd, int unlink) +{ + int i; + unsigned int niov; + int total_length = 0; + + lmd->md_me = NULL; + lmd->md_start = umd->start; + lmd->md_offset = 0; + lmd->md_max_size = umd->max_size; + lmd->md_options = umd->options; + lmd->md_user_ptr = umd->user_ptr; + lmd->md_eq = NULL; + lmd->md_threshold = umd->threshold; + lmd->md_refcount = 0; + lmd->md_flags = (unlink == LNET_UNLINK) ? LNET_MD_FLAG_AUTO_UNLINK : 0; + lmd->md_bulk_handle = umd->bulk_handle; - for (i = 0; i < (int)niov; i++) { - /* We take the page pointer on trust */ - if (lmd->md_iov.kiov[i].kiov_offset + - lmd->md_iov.kiov[i].kiov_len > PAGE_SIZE) - return -EINVAL; /* invalid length */ + if ((umd->options & LNET_MD_IOVEC) != 0) { + + if ((umd->options & LNET_MD_KIOV) != 0) /* Can't specify both */ + return -EINVAL; - total_length += lmd->md_iov.kiov[i].kiov_len; - } + lmd->md_niov = niov = umd->length; + memcpy(lmd->md_iov.iov, umd->start, + niov * sizeof(lmd->md_iov.iov[0])); - lmd->md_length = total_length; + for (i = 0; i < (int)niov; i++) { + /* We take the base address on trust */ + if (lmd->md_iov.iov[i].iov_len <= 0) /* invalid length */ + return -EINVAL; - if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ - (umd->max_size < 0 || - umd->max_size > total_length)) // illegal max_size - return -EINVAL; - } else { /* contiguous */ - lmd->md_length = umd->length; - lmd->md_niov = niov = 1; - lmd->md_iov.iov[0].iov_base = umd->start; - lmd->md_iov.iov[0].iov_len = umd->length; + total_length += lmd->md_iov.iov[i].iov_len; + } - if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ - (umd->max_size < 0 || - umd->max_size > (int)umd->length)) // illegal max_size - return -EINVAL; - } + lmd->md_length = total_length; + + if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ + (umd->max_size < 0 || + umd->max_size > total_length)) // illegal max_size + return -EINVAL; + + } else if ((umd->options & LNET_MD_KIOV) != 0) { + lmd->md_niov = niov = umd->length; + memcpy(lmd->md_iov.kiov, umd->start, + niov * sizeof(lmd->md_iov.kiov[0])); + + for (i = 0; i < (int)niov; i++) { + /* We take the page pointer on trust */ + if (lmd->md_iov.kiov[i].kiov_offset + + lmd->md_iov.kiov[i].kiov_len > PAGE_SIZE) + return -EINVAL; /* invalid length */ + + total_length += lmd->md_iov.kiov[i].kiov_len; + } + + lmd->md_length = total_length; + + if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ + (umd->max_size < 0 || + umd->max_size > total_length)) // illegal max_size + return -EINVAL; + } else { /* contiguous */ + lmd->md_length = umd->length; + lmd->md_niov = niov = 1; + lmd->md_iov.iov[0].iov_base = umd->start; + lmd->md_iov.iov[0].iov_len = umd->length; + + if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ + (umd->max_size < 0 || + umd->max_size > (int)umd->length)) // illegal max_size + return -EINVAL; + } return 0; } /* must be called with resource lock held */ static int -lnet_md_link(lnet_libmd_t *md, lnet_handle_eq_t eq_handle, int cpt) +lnet_md_link(struct lnet_libmd *md, struct lnet_handle_eq eq_handle, int cpt) { struct lnet_res_container *container = the_lnet.ln_md_containers[cpt]; @@ -172,14 +278,14 @@ lnet_md_link(lnet_libmd_t *md, lnet_handle_eq_t eq_handle, int cpt) * otherwise caller may only lnet_md_free() it. */ /* This implementation doesn't know how to create START events or - * disable END events. Best to LASSERT our caller is compliant so + * disable END events. Best to LASSERT our caller is compliant so * we find out quickly... */ /* TODO - reevaluate what should be here in light of * the removal of the start and end events * maybe there we shouldn't even allow LNET_EQ_NONE!) * LASSERT (eq == NULL); */ - if (!LNetHandleIsInvalid(eq_handle)) { + if (!LNetEQHandleIsInvalid(eq_handle)) { md->md_eq = lnet_handle2eq(&eq_handle); if (md->md_eq == NULL) @@ -198,40 +304,44 @@ lnet_md_link(lnet_libmd_t *md, lnet_handle_eq_t eq_handle, int cpt) /* must be called with lnet_res_lock held */ void -lnet_md_deconstruct(lnet_libmd_t *lmd, lnet_md_t *umd) +lnet_md_deconstruct(struct lnet_libmd *lmd, struct lnet_md *umd) { - /* NB this doesn't copy out all the iov entries so when a - * discontiguous MD is copied out, the target gets to know the - * original iov pointer (in start) and the number of entries it had - * and that's all. - */ - umd->start = lmd->md_start; - umd->length = ((lmd->md_options & (LNET_MD_IOVEC | LNET_MD_KIOV)) == 0) ? - lmd->md_length : lmd->md_niov; - umd->threshold = lmd->md_threshold; - umd->max_size = lmd->md_max_size; - umd->options = lmd->md_options; - umd->user_ptr = lmd->md_user_ptr; - lnet_eq2handle(&umd->eq_handle, lmd->md_eq); + /* NB this doesn't copy out all the iov entries so when a + * discontiguous MD is copied out, the target gets to know the + * original iov pointer (in start) and the number of entries it had + * and that's all. + */ + umd->start = lmd->md_start; + umd->length = ((lmd->md_options & (LNET_MD_IOVEC | LNET_MD_KIOV)) == 0) ? + lmd->md_length : lmd->md_niov; + umd->threshold = lmd->md_threshold; + umd->max_size = lmd->md_max_size; + umd->options = lmd->md_options; + umd->user_ptr = lmd->md_user_ptr; + lnet_eq2handle(&umd->eq_handle, lmd->md_eq); } static int -lnet_md_validate(lnet_md_t *umd) +lnet_md_validate(struct lnet_md *umd) { - if (umd->start == NULL && umd->length != 0) { - CERROR("MD start pointer can not be NULL with length %u\n", - umd->length); - return -EINVAL; - } - - if ((umd->options & (LNET_MD_KIOV | LNET_MD_IOVEC)) != 0 && - umd->length > LNET_MAX_IOV) { - CERROR("Invalid option: too many fragments %u, %d max\n", - umd->length, LNET_MAX_IOV); - return -EINVAL; - } - - return 0; + if (umd->start == NULL && umd->length != 0) { + CERROR("MD start pointer can not be NULL with length %u\n", + umd->length); + return -EINVAL; + } + + if ((umd->options & (LNET_MD_KIOV | LNET_MD_IOVEC)) != 0 && + umd->length > LNET_MAX_IOV) { + CERROR("Invalid option: too many fragments %u, %d max\n", + umd->length, LNET_MAX_IOV); + return -EINVAL; + } else if (umd->length > LNET_MTU) { + CERROR("Invalid length: too big fragment size %u, %d max\n", + umd->length, LNET_MTU); + return -EINVAL; + } + + return 0; } /** @@ -250,7 +360,7 @@ lnet_md_validate(lnet_md_t *umd) * \param handle On successful returns, a handle to the newly created MD is * saved here. This handle can be used later in LNetMDUnlink(). * - * \retval 0 On success. + * \retval 0 On success. * \retval -EINVAL If \a umd is not valid. * \retval -ENOMEM If new MD cannot be allocated. * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a @@ -260,29 +370,29 @@ lnet_md_validate(lnet_md_t *umd) * a MD. */ int -LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd, - lnet_unlink_t unlink, lnet_handle_md_t *handle) +LNetMDAttach(struct lnet_handle_me meh, struct lnet_md umd, + enum lnet_unlink unlink, struct lnet_handle_md *handle) { - struct list_head matches = LIST_HEAD_INIT(matches); - struct list_head drops = LIST_HEAD_INIT(drops); + LIST_HEAD(matches); + LIST_HEAD(drops); struct lnet_me *me; struct lnet_libmd *md; int cpt; int rc; - LASSERT (the_lnet.ln_refcount > 0); + LASSERT(the_lnet.ln_refcount > 0); - if (lnet_md_validate(&umd) != 0) - return -EINVAL; + if (lnet_md_validate(&umd) != 0) + return -EINVAL; - if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) == 0) { - CERROR("Invalid option: no MD_OP set\n"); - return -EINVAL; - } + if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) == 0) { + CERROR("Invalid option: no MD_OP set\n"); + return -EINVAL; + } - md = lnet_md_alloc(&umd); - if (md == NULL) - return -ENOMEM; + md = lnet_md_alloc(&umd); + if (md == NULL) + return -ENOMEM; rc = lnet_md_build(md, &umd, unlink); if (rc != 0) @@ -333,7 +443,7 @@ EXPORT_SYMBOL(LNetMDAttach); * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(), * and LNetGet() operations. * - * \retval 0 On success. + * \retval 0 On success. * \retval -EINVAL If \a umd is not valid. * \retval -ENOMEM If new MD cannot be allocated. * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that @@ -341,25 +451,26 @@ EXPORT_SYMBOL(LNetMDAttach); * LNetInvalidateHandle() on it. */ int -LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) +LNetMDBind(struct lnet_md umd, enum lnet_unlink unlink, + struct lnet_handle_md *handle) { - lnet_libmd_t *md; + struct lnet_libmd *md; int cpt; int rc; - LASSERT (the_lnet.ln_refcount > 0); + LASSERT(the_lnet.ln_refcount > 0); - if (lnet_md_validate(&umd) != 0) - return -EINVAL; + if (lnet_md_validate(&umd) != 0) + return -EINVAL; - if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) != 0) { - CERROR("Invalid option: GET|PUT illegal on active MDs\n"); - return -EINVAL; - } + if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) != 0) { + CERROR("Invalid option: GET|PUT illegal on active MDs\n"); + return -EINVAL; + } - md = lnet_md_alloc(&umd); - if (md == NULL) - return -ENOMEM; + md = lnet_md_alloc(&umd); + if (md == NULL) + return -ENOMEM; rc = lnet_md_build(md, &umd, unlink); if (rc != 0) @@ -412,15 +523,15 @@ EXPORT_SYMBOL(LNetMDBind); * * \param mdh A handle for the MD to be unlinked. * - * \retval 0 On success. + * \retval 0 On success. * \retval -ENOENT If \a mdh does not point to a valid MD object. */ int -LNetMDUnlink (lnet_handle_md_t mdh) +LNetMDUnlink(struct lnet_handle_md mdh) { - lnet_event_t ev; - lnet_libmd_t *md; - int cpt; + struct lnet_event ev; + struct lnet_libmd *md; + int cpt; LASSERT(the_lnet.ln_refcount > 0); @@ -430,8 +541,8 @@ LNetMDUnlink (lnet_handle_md_t mdh) md = lnet_handle2md(&mdh); if (md == NULL) { lnet_res_unlock(cpt); - return -ENOENT; - } + return -ENOENT; + } md->md_flags |= LNET_MD_FLAG_ABORTED; /* If the MD is busy, lnet_md_unlink just marks it for deletion, and @@ -442,7 +553,10 @@ LNetMDUnlink (lnet_handle_md_t mdh) lnet_eq_enqueue_event(md->md_eq, &ev); } - lnet_md_unlink(md); + if (md->md_rspt_ptr != NULL) + lnet_detach_rsp_tracker(md, cpt); + + lnet_md_unlink(md); lnet_res_unlock(cpt); return 0;