LU-13783 o2iblnd: make FMR-pool support optional.

Linux 5.8 removes the FMR-pool API.  This patch makes
all use for this API optional, selected only if the
support exists in the kernel.

Signed-off-by: Mr NeilBrown <neilb@suse.de>
Change-Id: I4c40f3a766f5b46ae4f26d7d3ecf8434a6e5a0cb
Reviewed-on: https://review.whamcloud.com/40287
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Sergey Gorenko <sergeygo@nvidia.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index 9554330..a9f05e3 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -109,8 +109,7 @@ AS_IF([test $ENABLEO2IB = "no"], [
        for O2IBPATH in $O2IBPATHS; do
                AS_IF([test \( -f ${O2IBPATH}/include/rdma/rdma_cm.h -a \
                           -f ${O2IBPATH}/include/rdma/ib_cm.h -a \
        for O2IBPATH in $O2IBPATHS; do
                AS_IF([test \( -f ${O2IBPATH}/include/rdma/rdma_cm.h -a \
                           -f ${O2IBPATH}/include/rdma/ib_cm.h -a \

-                          -f ${O2IBPATH}/include/rdma/ib_verbs.h -a \
-                          -f ${O2IBPATH}/include/rdma/ib_fmr_pool.h \)], [
+                          -f ${O2IBPATH}/include/rdma/ib_verbs.h \)], [

                        o2ib_found=true
                        break
                ])
                        o2ib_found=true
                        break
                ])


@@ -187,13 +186,11 @@ AS_IF([test $ENABLEO2IB = "no"], [
                        #include <rdma/rdma_cm.h>
                        #include <rdma/ib_cm.h>
                        #include <rdma/ib_verbs.h>
                        #include <rdma/rdma_cm.h>
                        #include <rdma/ib_cm.h>
                        #include <rdma/ib_verbs.h>

-                       #include <rdma/ib_fmr_pool.h>

                ],[
                        struct rdma_cm_id      *cm_idi __attribute__ ((unused));
                        struct rdma_conn_param  conn_param __attribute__ ((unused));
                        struct ib_device_attr   device_attr __attribute__ ((unused));
                        struct ib_qp_attr       qp_attr __attribute__ ((unused));
                ],[
                        struct rdma_cm_id      *cm_idi __attribute__ ((unused));
                        struct rdma_conn_param  conn_param __attribute__ ((unused));
                        struct ib_device_attr   device_attr __attribute__ ((unused));
                        struct ib_qp_attr       qp_attr __attribute__ ((unused));

-                       struct ib_pool_fmr      pool_fmr __attribute__ ((unused));

                        enum   ib_cm_rej_reason rej_reason __attribute__ ((unused));
                        rdma_destroy_id(NULL);
                ],[
                        enum   ib_cm_rej_reason rej_reason __attribute__ ((unused));
                        rdma_destroy_id(NULL);
                ],[


@@ -545,6 +542,18 @@ AS_IF([test $ENABLEO2IB != "no"], [
                        [rdma_reject has 4 arguments])
        ])
 
                        [rdma_reject has 4 arguments])
        ])
 

+       # The FMR pool API was removed in Linux 5.8,
+       # commit 4e373d5417ecbb4f438a8500f0379a2fc29c2643
+       LB_CHECK_COMPILE([if FMR pools API available],
+       ib_fmr, [
+               #include <rdma/ib_verbs.h>
+       ],[
+               struct ib_fmr fmr = {};
+       ],[
+               AC_DEFINE(HAVE_FMR_POOL_API, 1,
+                       [FMR pool API is available])
+       ])
+

        EXTRA_CHECK_INCLUDE=""
        AC_DEFUN([LN_CONFIG_O2IB_SRC], [])
        AC_DEFUN([LN_CONFIG_O2IB_RESULTS], [])
        EXTRA_CHECK_INCLUDE=""
        AC_DEFUN([LN_CONFIG_O2IB_SRC], [])
        AC_DEFUN([LN_CONFIG_O2IB_RESULTS], [])

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index d939510..18ac5fa 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1413,9 +1413,12 @@ kiblnd_destroy_fmr_pool(struct kib_fmr_pool *fpo)
 {
        LASSERT(fpo->fpo_map_count == 0);
 
 {
        LASSERT(fpo->fpo_map_count == 0);
 

+#ifdef HAVE_FMR_POOL_API

        if (fpo->fpo_is_fmr && fpo->fmr.fpo_fmr_pool) {
                ib_destroy_fmr_pool(fpo->fmr.fpo_fmr_pool);
        if (fpo->fpo_is_fmr && fpo->fmr.fpo_fmr_pool) {
                ib_destroy_fmr_pool(fpo->fmr.fpo_fmr_pool);

-       } else {
+       } else
+#endif /* HAVE_FMR_POOL_API */
+       {

                struct kib_fast_reg_descriptor *frd, *tmp;
                int i = 0;
 
                struct kib_fast_reg_descriptor *frd, *tmp;
                int i = 0;
 


@@ -1469,6 +1472,7 @@ kiblnd_fmr_flush_trigger(struct lnet_ioctl_config_o2iblnd_tunables *tunables,
        return max(IBLND_FMR_POOL_FLUSH, size);
 }
 
        return max(IBLND_FMR_POOL_FLUSH, size);
 }
 

+#ifdef HAVE_FMR_POOL_API

 static int kiblnd_alloc_fmr_pool(struct kib_fmr_poolset *fps,
                                 struct kib_fmr_pool *fpo)
 {
 static int kiblnd_alloc_fmr_pool(struct kib_fmr_poolset *fps,
                                 struct kib_fmr_pool *fpo)
 {


@@ -1497,6 +1501,7 @@ static int kiblnd_alloc_fmr_pool(struct kib_fmr_poolset *fps,
 
        return rc;
 }
 
        return rc;
 }

+#endif /* HAVE_FMR_POOL_API */

 
 static int kiblnd_alloc_freg_pool(struct kib_fmr_poolset *fps,
                                  struct kib_fmr_pool *fpo,
 
 static int kiblnd_alloc_freg_pool(struct kib_fmr_poolset *fps,
                                  struct kib_fmr_pool *fpo,


@@ -1505,7 +1510,9 @@ static int kiblnd_alloc_freg_pool(struct kib_fmr_poolset *fps,
        struct kib_fast_reg_descriptor *frd, *tmp;
        int i, rc;
 
        struct kib_fast_reg_descriptor *frd, *tmp;
        int i, rc;
 

+#ifdef HAVE_FMR_POOL_API

        fpo->fpo_is_fmr = false;
        fpo->fpo_is_fmr = false;

+#endif

 
        INIT_LIST_HEAD(&fpo->fast_reg.fpo_pool_list);
        fpo->fast_reg.fpo_pool_size = 0;
 
        INIT_LIST_HEAD(&fpo->fast_reg.fpo_pool_list);
        fpo->fast_reg.fpo_pool_size = 0;


@@ -1612,9 +1619,11 @@ static int kiblnd_create_fmr_pool(struct kib_fmr_poolset *fps,
 
        fpo->fpo_hdev = kiblnd_current_hdev(dev);
 
 
        fpo->fpo_hdev = kiblnd_current_hdev(dev);
 

+#ifdef HAVE_FMR_POOL_API

        if (dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED)
                rc = kiblnd_alloc_fmr_pool(fps, fpo);
        else
        if (dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED)
                rc = kiblnd_alloc_fmr_pool(fps, fpo);
        else

+#endif /* HAVE_FMR_POOL_API */

                rc = kiblnd_alloc_freg_pool(fps, fpo, dev->ibd_dev_caps);
        if (rc)
                goto out_fpo;
                rc = kiblnd_alloc_freg_pool(fps, fpo, dev->ibd_dev_caps);
        if (rc)
                goto out_fpo;


@@ -1701,6 +1710,7 @@ kiblnd_fmr_pool_is_idle(struct kib_fmr_pool *fpo, time64_t now)
        return now >= fpo->fpo_deadline;
 }
 
        return now >= fpo->fpo_deadline;
 }
 

+#if defined(HAVE_FMR_POOL_API) || !defined(HAVE_IB_MAP_MR_SG)

 static int
 kiblnd_map_tx_pages(struct kib_tx *tx, struct kib_rdma_desc *rd)
 {
 static int
 kiblnd_map_tx_pages(struct kib_tx *tx, struct kib_rdma_desc *rd)
 {


@@ -1722,6 +1732,7 @@ kiblnd_map_tx_pages(struct kib_tx *tx, struct kib_rdma_desc *rd)
 
        return npages;
 }
 
        return npages;
 }

+#endif

 
 void
 kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 
 void
 kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)


@@ -1731,12 +1742,13 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
        struct kib_fmr_poolset *fps;
        time64_t now = ktime_get_seconds();
        struct kib_fmr_pool *tmp;
        struct kib_fmr_poolset *fps;
        time64_t now = ktime_get_seconds();
        struct kib_fmr_pool *tmp;

-       int rc;

 
        if (!fpo)
                return;
 
        fps = fpo->fpo_owner;
 
        if (!fpo)
                return;
 
        fps = fpo->fpo_owner;

+
+#ifdef HAVE_FMR_POOL_API

        if (fpo->fpo_is_fmr) {
                if (fmr->fmr_pfmr) {
                        ib_fmr_pool_unmap(fmr->fmr_pfmr);
        if (fpo->fpo_is_fmr) {
                if (fmr->fmr_pfmr) {
                        ib_fmr_pool_unmap(fmr->fmr_pfmr);


@@ -1744,10 +1756,12 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
                }
 
                if (status) {
                }
 
                if (status) {

-                       rc = ib_flush_fmr_pool(fpo->fmr.fpo_fmr_pool);
+                       int rc = ib_flush_fmr_pool(fpo->fmr.fpo_fmr_pool);

                        LASSERT(!rc);
                }
                        LASSERT(!rc);
                }

-       } else {
+       } else
+#endif /* HAVE_FMR_POOL_API */
+       {

                struct kib_fast_reg_descriptor *frd = fmr->fmr_frd;
 
                if (frd) {
                struct kib_fast_reg_descriptor *frd = fmr->fmr_frd;
 
                if (frd) {


@@ -1784,11 +1798,13 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
                        struct kib_fmr *fmr)
 {
        struct kib_fmr_pool *fpo;
                        struct kib_fmr *fmr)
 {
        struct kib_fmr_pool *fpo;

-       __u64 *pages = tx->tx_pages;

        __u64 version;
        bool is_rx = (rd != tx->tx_rd);
        __u64 version;
        bool is_rx = (rd != tx->tx_rd);

+#ifdef HAVE_FMR_POOL_API
+       __u64 *pages = tx->tx_pages;

        bool tx_pages_mapped = 0;
        int npages = 0;
        bool tx_pages_mapped = 0;
        int npages = 0;

+#endif

        int rc;
 
 again:
        int rc;
 
 again:


@@ -1798,6 +1814,8 @@ again:
                fpo->fpo_deadline = ktime_get_seconds() + IBLND_POOL_DEADLINE;
                fpo->fpo_map_count++;
 
                fpo->fpo_deadline = ktime_get_seconds() + IBLND_POOL_DEADLINE;
                fpo->fpo_map_count++;
 

+#ifdef HAVE_FMR_POOL_API
+               fmr->fmr_pfmr = NULL;

                if (fpo->fpo_is_fmr) {
                        struct ib_pool_fmr *pfmr;
 
                if (fpo->fpo_is_fmr) {
                        struct ib_pool_fmr *pfmr;
 


@@ -1819,7 +1837,9 @@ again:
                                return 0;
                        }
                        rc = PTR_ERR(pfmr);
                                return 0;
                        }
                        rc = PTR_ERR(pfmr);

-               } else {
+               } else
+#endif /* HAVE_FMR_POOL_API */
+               {

                        if (!list_empty(&fpo->fast_reg.fpo_pool_list)) {
                                struct kib_fast_reg_descriptor *frd;
 #ifdef HAVE_IB_MAP_MR_SG
                        if (!list_empty(&fpo->fast_reg.fpo_pool_list)) {
                                struct kib_fast_reg_descriptor *frd;
 #ifdef HAVE_IB_MAP_MR_SG


@@ -1865,7 +1885,7 @@ again:
 #else
                                n = ib_map_mr_sg(mr, tx->tx_frags,
                                                 rd->rd_nfrags, PAGE_SIZE);
 #else
                                n = ib_map_mr_sg(mr, tx->tx_frags,
                                                 rd->rd_nfrags, PAGE_SIZE);

-#endif
+#endif /* HAVE_IB_MAP_MR_SG_5ARGS */

                                if (unlikely(n != rd->rd_nfrags)) {
                                        CERROR("Failed to map mr %d/%d "
                                               "elements\n", n, rd->rd_nfrags);
                                if (unlikely(n != rd->rd_nfrags)) {
                                        CERROR("Failed to map mr %d/%d "
                                               "elements\n", n, rd->rd_nfrags);


@@ -1883,7 +1903,7 @@ again:
                                wr->key = is_rx ? mr->rkey : mr->lkey;
                                wr->access = (IB_ACCESS_LOCAL_WRITE |
                                              IB_ACCESS_REMOTE_WRITE);
                                wr->key = is_rx ? mr->rkey : mr->lkey;
                                wr->access = (IB_ACCESS_LOCAL_WRITE |
                                              IB_ACCESS_REMOTE_WRITE);

-#else
+#else /* HAVE_IB_MAP_MR_SG */

                                if (!tx_pages_mapped) {
                                        npages = kiblnd_map_tx_pages(tx, rd);
                                        tx_pages_mapped = 1;
                                if (!tx_pages_mapped) {
                                        npages = kiblnd_map_tx_pages(tx, rd);
                                        tx_pages_mapped = 1;


@@ -1910,11 +1930,10 @@ again:
                                wr->wr.wr.fast_reg.access_flags =
                                                (IB_ACCESS_LOCAL_WRITE |
                                                 IB_ACCESS_REMOTE_WRITE);
                                wr->wr.wr.fast_reg.access_flags =
                                                (IB_ACCESS_LOCAL_WRITE |
                                                 IB_ACCESS_REMOTE_WRITE);

-#endif
+#endif /* HAVE_IB_MAP_MR_SG */

 
                                fmr->fmr_key  = is_rx ? mr->rkey : mr->lkey;
                                fmr->fmr_frd  = frd;
 
                                fmr->fmr_key  = is_rx ? mr->rkey : mr->lkey;
                                fmr->fmr_frd  = frd;

-                               fmr->fmr_pfmr = NULL;

                                fmr->fmr_pool = fpo;
                                return 0;
                        }
                                fmr->fmr_pool = fpo;
                                return 0;
                        }


@@ -2588,6 +2607,7 @@ kiblnd_hdev_get_attr(struct kib_hca_dev *hdev)
        hdev->ibh_max_qp_wr = dev_attr->max_qp_wr;
 
        /* Setup device Memory Registration capabilities */
        hdev->ibh_max_qp_wr = dev_attr->max_qp_wr;
 
        /* Setup device Memory Registration capabilities */

+#ifdef HAVE_FMR_POOL_API

 #ifdef HAVE_IB_DEVICE_OPS
        if (hdev->ibh_ibdev->ops.alloc_fmr &&
            hdev->ibh_ibdev->ops.dealloc_fmr &&
 #ifdef HAVE_IB_DEVICE_OPS
        if (hdev->ibh_ibdev->ops.alloc_fmr &&
            hdev->ibh_ibdev->ops.dealloc_fmr &&


@@ -2601,7 +2621,9 @@ kiblnd_hdev_get_attr(struct kib_hca_dev *hdev)
 #endif
                LCONSOLE_INFO("Using FMR for registration\n");
                hdev->ibh_dev->ibd_dev_caps |= IBLND_DEV_CAPS_FMR_ENABLED;
 #endif
                LCONSOLE_INFO("Using FMR for registration\n");
                hdev->ibh_dev->ibd_dev_caps |= IBLND_DEV_CAPS_FMR_ENABLED;

-       } else if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
+       } else
+#endif /* HAVE_FMR_POOL_API */
+       if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {

                LCONSOLE_INFO("Using FastReg for registration\n");
                hdev->ibh_dev->ibd_dev_caps |= IBLND_DEV_CAPS_FASTREG_ENABLED;
 #ifndef HAVE_IB_ALLOC_FAST_REG_MR
                LCONSOLE_INFO("Using FastReg for registration\n");
                hdev->ibh_dev->ibd_dev_caps |= IBLND_DEV_CAPS_FASTREG_ENABLED;
 #ifndef HAVE_IB_ALLOC_FAST_REG_MR

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 2f7573c..b8061f2 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -83,7 +83,9 @@
 #include <rdma/rdma_cm.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_verbs.h>

+#ifdef HAVE_FMR_POOL_API

 #include <rdma/ib_fmr_pool.h>
 #include <rdma/ib_fmr_pool.h>

+#endif

 
 #define DEBUG_SUBSYSTEM S_LND
 
 
 #define DEBUG_SUBSYSTEM S_LND
 


@@ -179,7 +181,9 @@ struct kib_hca_dev;
 enum kib_dev_caps {
        IBLND_DEV_CAPS_FASTREG_ENABLED          = BIT(0),
        IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT     = BIT(1),
 enum kib_dev_caps {
        IBLND_DEV_CAPS_FASTREG_ENABLED          = BIT(0),
        IBLND_DEV_CAPS_FASTREG_GAPS_SUPPORT     = BIT(1),

+#ifdef HAVE_FMR_POOL_API

        IBLND_DEV_CAPS_FMR_ENABLED              = BIT(2),
        IBLND_DEV_CAPS_FMR_ENABLED              = BIT(2),

+#endif

 };
 
 struct kib_dev {
 };
 
 struct kib_dev {


@@ -346,28 +350,36 @@ struct kib_fmr_pool {
        struct list_head        fpo_list;       /* chain on pool list */
        struct kib_hca_dev     *fpo_hdev;       /* device for this pool */
        struct kib_fmr_poolset      *fpo_owner; /* owner of this pool */
        struct list_head        fpo_list;       /* chain on pool list */
        struct kib_hca_dev     *fpo_hdev;       /* device for this pool */
        struct kib_fmr_poolset      *fpo_owner; /* owner of this pool */

+#ifdef HAVE_FMR_POOL_API

        union {
                struct {
                        struct ib_fmr_pool *fpo_fmr_pool; /* IB FMR pool */
                } fmr;
        union {
                struct {
                        struct ib_fmr_pool *fpo_fmr_pool; /* IB FMR pool */
                } fmr;

+#endif

                struct { /* For fast registration */
                        struct list_head  fpo_pool_list;
                        int               fpo_pool_size;
                } fast_reg;
                struct { /* For fast registration */
                        struct list_head  fpo_pool_list;
                        int               fpo_pool_size;
                } fast_reg;

+#ifdef HAVE_FMR_POOL_API

        };
        };

+       bool                    fpo_is_fmr; /* True if FMR pools allocated */
+#endif

        time64_t                fpo_deadline;   /* deadline of this pool */
        int                     fpo_failed;     /* fmr pool is failed */
        int                     fpo_map_count;  /* # of mapped FMR */
        time64_t                fpo_deadline;   /* deadline of this pool */
        int                     fpo_failed;     /* fmr pool is failed */
        int                     fpo_map_count;  /* # of mapped FMR */

-       bool                    fpo_is_fmr; /* True if FMR pools allocated */

 };
 
 struct kib_fmr {
        struct kib_fmr_pool             *fmr_pool;      /* pool of FMR */
 };
 
 struct kib_fmr {
        struct kib_fmr_pool             *fmr_pool;      /* pool of FMR */

+#ifdef HAVE_FMR_POOL_API

        struct ib_pool_fmr              *fmr_pfmr;      /* IB pool fmr */
        struct ib_pool_fmr              *fmr_pfmr;      /* IB pool fmr */

+#endif /* HAVE_FMR_POOL_API */

        struct kib_fast_reg_descriptor  *fmr_frd;
        u32                              fmr_key;
 };
 
        struct kib_fast_reg_descriptor  *fmr_frd;
        u32                              fmr_key;
 };
 

+#ifdef HAVE_FMR_POOL_API
+

 #ifdef HAVE_ORACLE_OFED_EXTENSIONS
 #define kib_fmr_pool_map(pool, pgs, n, iov) \
        ib_fmr_pool_map_phys((pool), (pgs), (n), (iov), NULL)
 #ifdef HAVE_ORACLE_OFED_EXTENSIONS
 #define kib_fmr_pool_map(pool, pgs, n, iov) \
        ib_fmr_pool_map_phys((pool), (pgs), (n), (iov), NULL)


@@ -376,6 +388,8 @@ struct kib_fmr {
        ib_fmr_pool_map_phys((pool), (pgs), (n), (iov))
 #endif
 
        ib_fmr_pool_map_phys((pool), (pgs), (n), (iov))
 #endif
 

+#endif /* HAVE_FMR_POOL_API */
+

 struct kib_net {
        /* chain on struct kib_dev::ibd_nets */
        struct list_head        ibn_list;
 struct kib_net {
        /* chain on struct kib_dev::ibd_nets */
        struct list_head        ibn_list;

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 0960c25..055b20b 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -588,6 +588,7 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
                return -EPROTONOSUPPORT;
        }
 
                return -EPROTONOSUPPORT;
        }
 

+#ifdef HAVE_FMR_POOL_API

        /*
         * FMR does not support gaps but the tx has gaps then
         * we should make sure that the number of fragments we'll be sending
        /*
         * FMR does not support gaps but the tx has gaps then
         * we should make sure that the number of fragments we'll be sending


@@ -606,6 +607,7 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
                        return -EFBIG;
                }
        }
                        return -EFBIG;
                }
        }

+#endif

 
        fps = net->ibn_fmr_ps[cpt];
        rc = kiblnd_fmr_pool_map(fps, tx, rd, nob, 0, &tx->tx_fmr);
 
        fps = net->ibn_fmr_ps[cpt];
        rc = kiblnd_fmr_pool_map(fps, tx, rd, nob, 0, &tx->tx_fmr);


@@ -624,11 +626,17 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
         * for FastReg or FMR with no gaps we can accumulate all
         * the fragments in one FastReg or FMR fragment.
         */
         * for FastReg or FMR with no gaps we can accumulate all
         * the fragments in one FastReg or FMR fragment.
         */

-       if (((dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED) && !tx->tx_gaps) ||
+       if (
+#ifdef HAVE_FMR_POOL_API
+           ((dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED)
+            && !tx->tx_gaps) ||
+#endif

            (dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED)) {
                /* FMR requires zero based address */
            (dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED)) {
                /* FMR requires zero based address */

+#ifdef HAVE_FMR_POOL_API

                if (dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED)
                        rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask;
                if (dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED)
                        rd->rd_frags[0].rf_addr &= ~hdev->ibh_page_mask;

+#endif

                rd->rd_frags[0].rf_nob = nob;
                rd->rd_nfrags = 1;
        } else {
                rd->rd_frags[0].rf_nob = nob;
                rd->rd_nfrags = 1;
        } else {


@@ -649,7 +657,11 @@ kiblnd_fmr_map_tx(struct kib_net *net, struct kib_tx *tx,
 static void
 kiblnd_unmap_tx(struct kib_tx *tx)
 {
 static void
 kiblnd_unmap_tx(struct kib_tx *tx)
 {

-       if (tx->tx_fmr.fmr_pfmr || tx->tx_fmr.fmr_frd)
+       if (
+#ifdef HAVE_FMR_POOL_API
+               tx->tx_fmr.fmr_pfmr ||
+#endif
+               tx->tx_fmr.fmr_frd)

                kiblnd_fmr_pool_unmap(&tx->tx_fmr, tx->tx_status);
 
        if (tx->tx_nfrags != 0) {
                kiblnd_fmr_pool_unmap(&tx->tx_fmr, tx->tx_status);
 
        if (tx->tx_nfrags != 0) {


@@ -676,8 +688,11 @@ kiblnd_find_rd_dma_mr(struct lnet_ni *ni, struct kib_rdma_desc *rd)
         * dead in the water and fail the operation.
         */
        if (tunables->lnd_map_on_demand &&
         * dead in the water and fail the operation.
         */
        if (tunables->lnd_map_on_demand &&

-           (net->ibn_dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED ||
-            net->ibn_dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED))
+           (net->ibn_dev->ibd_dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED
+#ifdef HAVE_FMR_POOL_API
+            || net->ibn_dev->ibd_dev_caps & IBLND_DEV_CAPS_FMR_ENABLED
+#endif
+       ))

                return NULL;
 
        /*
                return NULL;
 
        /*