From f4bc1a894ae156167154e91c080beae5a17b6d0e Mon Sep 17 00:00:00 2001 From: eeb Date: Fri, 21 Oct 2005 16:15:45 +0000 Subject: [PATCH] * Changed iiblnd to use Infinicon iba_xxx() API. * Changed iiblnd to post multiple work items in 1 go. --- lnet/klnds/iiblnd/iiblnd.c | 139 ++++++++++---------- lnet/klnds/iiblnd/iiblnd.h | 290 +----------------------------------------- lnet/klnds/iiblnd/iiblnd_cb.c | 73 ++++++----- 3 files changed, 105 insertions(+), 397 deletions(-) diff --git a/lnet/klnds/iiblnd/iiblnd.c b/lnet/klnds/iiblnd/iiblnd.c index e78a06c..c6dfb94 100644 --- a/lnet/klnds/iiblnd/iiblnd.c +++ b/lnet/klnds/iiblnd/iiblnd.c @@ -307,7 +307,7 @@ kibnal_create_cep(lnet_nid_t nid) __u32 u32val; IB_HANDLE cep; - cep = iibt_cm_create_cep(CM_RC_TYPE); + cep = iba_cm_create_cep(CM_RC_TYPE); if (cep == NULL) { CERROR ("Can't create CEP for %s\n", (nid == LNET_NID_ANY) ? "listener" : @@ -317,16 +317,16 @@ kibnal_create_cep(lnet_nid_t nid) if (nid == LNET_NID_ANY) { u32val = 1; - frc = iibt_cm_modify_cep(cep, CM_FLAG_ASYNC_ACCEPT, - (char *)&u32val, sizeof(u32val), 0); + frc = iba_cm_modify_cep(cep, CM_FLAG_ASYNC_ACCEPT, + (char *)&u32val, sizeof(u32val), 0); if (frc != FSUCCESS) { CERROR("Can't set async_accept: %d\n", frc); goto failed; } u32val = 0; /* sets system max */ - frc = iibt_cm_modify_cep(cep, CM_FLAG_LISTEN_BACKLOG, - (char *)&u32val, sizeof(u32val), 0); + frc = iba_cm_modify_cep(cep, CM_FLAG_LISTEN_BACKLOG, + (char *)&u32val, sizeof(u32val), 0); if (frc != FSUCCESS) { CERROR("Can't set listen backlog: %d\n", frc); goto failed; @@ -334,8 +334,8 @@ kibnal_create_cep(lnet_nid_t nid) } u32val = 1; - frc = iibt_cm_modify_cep(cep, CM_FLAG_TIMEWAIT_CALLBACK, - (char *)&u32val, sizeof(u32val), 0); + frc = iba_cm_modify_cep(cep, CM_FLAG_TIMEWAIT_CALLBACK, + (char *)&u32val, sizeof(u32val), 0); if (frc != FSUCCESS) { CERROR("Can't set timewait_callback for %s: %d\n", (nid == LNET_NID_ANY) ? "listener" : @@ -346,7 +346,7 @@ kibnal_create_cep(lnet_nid_t nid) return cep; failed: - iibt_cm_destroy_cep(cep); + iba_cm_destroy_cep(cep); return NULL; } @@ -441,12 +441,12 @@ kibnal_check_advert (void) kibnal_data.kib_ni->ni_nid); qry.InputValue.ServiceRecordValue.ComponentMask = KIBNAL_SERVICE_KEY_MASK; - frc = iibt_sd_query_port_fabric_information(kibnal_data.kib_sd, - kibnal_data.kib_port_guid, - &qry, - kibnal_service_query_done, - &kibnal_data.kib_sdretry, - &rc); + frc = iba_sd_query_port_fabric_info(kibnal_data.kib_sd, + kibnal_data.kib_port_guid, + &qry, + kibnal_service_query_done, + &kibnal_data.kib_sdretry, + &rc); if (frc != FPENDING) { CERROR ("Immediate error %d checking SM service\n", frc); return -EIO; @@ -516,12 +516,12 @@ kibnal_advertise (void) svc->RID.ServiceID, svc->ServiceName, libcfs_nid2str(le64_to_cpu(*kibnal_service_nid_field(svc)))); - frc = iibt_sd_port_fabric_operation(kibnal_data.kib_sd, - kibnal_data.kib_port_guid, - &fod, - kibnal_service_setunset_done, - &kibnal_data.kib_sdretry, - &frc2); + frc = iba_sd_port_fabric_operation(kibnal_data.kib_sd, + kibnal_data.kib_port_guid, + &fod, + kibnal_service_setunset_done, + &kibnal_data.kib_sdretry, + &frc2); if (frc != FSUCCESS && frc != FPENDING) { CERROR ("Immediate error %d advertising NID %s\n", @@ -558,12 +558,12 @@ kibnal_unadvertise (int expect_success) svc->ServiceName, libcfs_nid2str(le64_to_cpu(*kibnal_service_nid_field(svc)))); - frc = iibt_sd_port_fabric_operation(kibnal_data.kib_sd, - kibnal_data.kib_port_guid, - &fod, - kibnal_service_setunset_done, - &kibnal_data.kib_sdretry, - &frc2); + frc = iba_sd_port_fabric_operation(kibnal_data.kib_sd, + kibnal_data.kib_port_guid, + &fod, + kibnal_service_setunset_done, + &kibnal_data.kib_sdretry, + &frc2); if (frc != FSUCCESS && frc != FPENDING) { CERROR ("Immediate error %d unadvertising NID %s\n", frc, libcfs_nid2str(kibnal_data.kib_ni->ni_nid)); @@ -598,13 +598,13 @@ kibnal_stop_listener(int normal_shutdown) kibnal_unadvertise(normal_shutdown); - frc = iibt_cm_cancel(cep); + frc = iba_cm_cancel(cep); if (frc != FSUCCESS && frc != FPENDING) CERROR ("Error %d stopping listener\n", frc); down(&kibnal_data.kib_listener_signal); - frc = iibt_cm_destroy_cep(cep); + frc = iba_cm_destroy_cep(cep); if (frc != FSUCCESS) CERROR ("Error %d destroying listener CEP\n", frc); @@ -639,11 +639,11 @@ kibnal_start_listener(void) memset (&info, 0, sizeof(info)); info.ListenAddr.EndPt.SID = *kibnal_tunables.kib_service_number; - frc = iibt_cm_listen(cep, &info, kibnal_listen_callback, NULL); + frc = iba_cm_listen(cep, &info, kibnal_listen_callback, NULL); if (frc != FSUCCESS && frc != FPENDING) { - CERROR ("iibt_cm_listen error: %d\n", frc); + CERROR ("iba_cm_listen error: %d\n", frc); - iibt_cm_destroy_cep(cep); + iba_cm_destroy_cep(cep); return -EIO; } @@ -977,7 +977,7 @@ kibnal_conn_rts(kib_conn_t *conn, GetAVFromPath(0, path, &modify_attr.PathMTU, NULL, &modify_attr.DestAV); - frc = iibt_qp_modify(qp, &modify_attr, NULL); + frc = iba_modify_qp(qp, &modify_attr, NULL); if (frc != FSUCCESS) { CERROR("Can't set QP %s ready to receive: %d\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), frc); @@ -1007,14 +1007,14 @@ kibnal_conn_rts(kib_conn_t *conn, IB_QP_ATTR_RNRRETRYCOUNT), }; - frc = iibt_qp_modify(qp, &modify_attr, NULL); + frc = iba_modify_qp(qp, &modify_attr, NULL); if (frc != FSUCCESS) { CERROR("Can't set QP %s ready to send: %d\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), frc); return -EIO; } - frc = iibt_qp_query(conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs, NULL); + frc = iba_query_qp(conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs, NULL); if (frc != FSUCCESS) { CERROR ("Can't query QP %s attributes: %d\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), frc); @@ -1113,8 +1113,8 @@ kibnal_create_conn (lnet_nid_t nid) .PDHandle = kibnal_data.kib_pd, .SendSignaledCompletions = TRUE, }; - frc = iibt_qp_create(kibnal_data.kib_hca, ¶ms.qp_create, NULL, - &conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs); + frc = iba_create_qp(kibnal_data.kib_hca, ¶ms.qp_create, NULL, + &conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs); if (frc != 0) { CERROR ("Can't create QP %s: %d\n", libcfs_nid2str(nid), frc); goto failed; @@ -1137,14 +1137,14 @@ kibnal_create_conn (lnet_nid_t nid) }, }, }; - frc = iibt_qp_modify(conn->ibc_qp, ¶ms.qp_attr, NULL); + frc = iba_modify_qp(conn->ibc_qp, ¶ms.qp_attr, NULL); if (frc != 0) { CERROR ("Can't set QP %s state to INIT: %d\n", libcfs_nid2str(nid), frc); goto failed; } - frc = iibt_qp_query(conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs, NULL); + frc = iba_query_qp(conn->ibc_qp, &conn->ibc_cvars->cv_qpattrs, NULL); if (frc != FSUCCESS) { CERROR ("Can't query QP %s attributes: %d\n", libcfs_nid2str(nid), frc); @@ -1153,7 +1153,7 @@ kibnal_create_conn (lnet_nid_t nid) /* 1 ref for caller */ atomic_set (&conn->ibc_refcount, 1); - CDEBUG(D_WARNING, "New conn %p\n", conn); + CDEBUG(D_NET, "New conn %p\n", conn); return (conn); failed: @@ -1194,14 +1194,14 @@ kibnal_destroy_conn (kib_conn_t *conn) } if (conn->ibc_cep != NULL) { - frc = iibt_cm_destroy_cep(conn->ibc_cep); + frc = iba_cm_destroy_cep(conn->ibc_cep); if (frc != FSUCCESS) CERROR("Error destroying CEP %p: %d\n", conn->ibc_cep, frc); } if (conn->ibc_qp != NULL) { - frc = iibt_qp_destroy(conn->ibc_qp); + frc = iba_destroy_qp(conn->ibc_qp); if (frc != FSUCCESS) CERROR("Error destroying QP %p: %d\n", conn->ibc_qp, frc); @@ -1588,20 +1588,19 @@ kibnal_register_all_memory(void) phys.PhysAddr = 0; phys.Length = total; - frc = iibt_register_contig_physical_memory( - kibnal_data.kib_hca, 0, &phys, 1, 0, - kibnal_data.kib_pd, access, - &kibnal_data.kib_whole_mem.md_handle, - &kibnal_data.kib_whole_mem.md_addr, - &kibnal_data.kib_whole_mem.md_lkey, - &kibnal_data.kib_whole_mem.md_rkey); + frc = iba_register_contig_pmr(kibnal_data.kib_hca, 0, &phys, 1, 0, + kibnal_data.kib_pd, access, + &kibnal_data.kib_whole_mem.md_handle, + &kibnal_data.kib_whole_mem.md_addr, + &kibnal_data.kib_whole_mem.md_lkey, + &kibnal_data.kib_whole_mem.md_rkey); if (frc != FSUCCESS) { CERROR("registering physical memory failed: %d\n", frc); return -EIO; } - CDEBUG(D_WARNING, "registered phys mem from "LPX64" for "LPU64"\n", + CDEBUG(D_NET, "registered phys mem from "LPX64" for "LPU64"\n", phys.PhysAddr, phys.Length); return 0; @@ -1642,7 +1641,7 @@ kibnal_shutdown (lnet_ni_t *ni) /* fall through */ case IBNAL_INIT_CQ: - rc = iibt_cq_destroy(kibnal_data.kib_cq); + rc = iba_destroy_cq(kibnal_data.kib_cq); if (rc != 0) CERROR ("Destroy CQ error: %d\n", rc); /* fall through */ @@ -1652,19 +1651,19 @@ kibnal_shutdown (lnet_ni_t *ni) /* fall through */ case IBNAL_INIT_MD: - rc = iibt_deregister_memory(kibnal_data.kib_whole_mem.md_handle); + rc = iba_deregister_mr(kibnal_data.kib_whole_mem.md_handle); if (rc != FSUCCESS) CERROR ("Deregister memory: %d\n", rc); /* fall through */ case IBNAL_INIT_PD: - rc = iibt_pd_free(kibnal_data.kib_pd); + rc = iba_free_pd(kibnal_data.kib_pd); if (rc != 0) CERROR ("Destroy PD error: %d\n", rc); /* fall through */ case IBNAL_INIT_SD: - rc = iibt_sd_deregister(kibnal_data.kib_sd); + rc = iba_sd_deregister(kibnal_data.kib_sd); if (rc != 0) CERROR ("Deregister SD error: %d\n", rc); /* fall through */ @@ -1675,7 +1674,7 @@ kibnal_shutdown (lnet_ni_t *ni) /* fall through */ case IBNAL_INIT_HCA: - rc = iibt_close_hca(kibnal_data.kib_hca); + rc = iba_close_ca(kibnal_data.kib_hca); if (rc != 0) CERROR ("Close HCA error: %d\n", rc); /* fall through */ @@ -1813,14 +1812,6 @@ kibnal_startup (lnet_ni_t *ni) do_gettimeofday(&tv); kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec; - frc = IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2, - &kibnal_data.kib_interfaces); - if (frc != FSUCCESS) { - CERROR("IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2) = %d\n", - frc); - return -ENOSYS; - } - PORTAL_MODULE_USE; rwlock_init(&kibnal_data.kib_global_lock); @@ -1878,7 +1869,7 @@ kibnal_startup (lnet_ni_t *ni) n = sizeof(kibnal_data.kib_hca_guids) / sizeof(kibnal_data.kib_hca_guids[0]); - frc = iibt_get_hca_guids(&n, kibnal_data.kib_hca_guids); + frc = iba_get_caguids(&n, kibnal_data.kib_hca_guids); if (frc != FSUCCESS) { CERROR ("Can't get HCA guids: %d\n", frc); goto failed; @@ -1896,7 +1887,7 @@ kibnal_startup (lnet_ni_t *ni) } /* Infinicon has per-HCA notification callbacks */ - frc = iibt_open_hca(kibnal_data.kib_hca_guids[kibnal_data.kib_hca_idx], + frc = iba_open_ca(kibnal_data.kib_hca_guids[kibnal_data.kib_hca_idx], kibnal_hca_callback, kibnal_hca_async_callback, NULL, @@ -1913,8 +1904,8 @@ kibnal_startup (lnet_ni_t *ni) kibnal_data.kib_hca_attrs.PortAttributesList = NULL; kibnal_data.kib_hca_attrs.PortAttributesListSize = 0; - frc = iibt_query_hca(kibnal_data.kib_hca, - &kibnal_data.kib_hca_attrs, NULL); + frc = iba_query_ca(kibnal_data.kib_hca, + &kibnal_data.kib_hca_attrs, NULL); if (frc != FSUCCESS) { CERROR ("Can't size port attrs: %d\n", frc); goto failed; @@ -1929,8 +1920,8 @@ kibnal_startup (lnet_ni_t *ni) kibnal_data.kib_init = IBNAL_INIT_PORTATTRS; /*****************************************************/ - frc = iibt_query_hca(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs, - NULL); + frc = iba_query_ca(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs, + NULL); if (frc != FSUCCESS) { CERROR ("Can't get port attrs for HCA %d: %d\n", kibnal_data.kib_hca_idx, frc); @@ -1972,7 +1963,7 @@ kibnal_startup (lnet_ni_t *ni) CDEBUG(D_NET, "got guid "LPX64"\n", kibnal_data.kib_port_guid); - frc = iibt_sd_register(&kibnal_data.kib_sd, NULL); + frc = iba_sd_register(&kibnal_data.kib_sd, NULL); if (frc != FSUCCESS) { CERROR ("Can't register with SD: %d\n", frc); goto failed; @@ -1982,7 +1973,7 @@ kibnal_startup (lnet_ni_t *ni) kibnal_data.kib_init = IBNAL_INIT_SD; /*****************************************************/ - frc = iibt_pd_allocate(kibnal_data.kib_hca, 0, &kibnal_data.kib_pd); + frc = iba_alloc_pd(kibnal_data.kib_hca, 0, &kibnal_data.kib_pd); if (frc != FSUCCESS) { CERROR ("Can't create PD: %d\n", rc); goto failed; @@ -2012,9 +2003,9 @@ kibnal_startup (lnet_ni_t *ni) kibnal_data.kib_init = IBNAL_INIT_TXD; /*****************************************************/ - frc = iibt_cq_create(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES(), - &kibnal_data.kib_cq, &kibnal_data.kib_cq, - &n); + frc = iba_create_cq(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES(), + &kibnal_data.kib_cq, &kibnal_data.kib_cq, + &n); if (frc != FSUCCESS) { CERROR ("Can't create RX CQ: %d\n", frc); goto failed; @@ -2030,7 +2021,7 @@ kibnal_startup (lnet_ni_t *ni) goto failed; } - rc = iibt_cq_rearm(kibnal_data.kib_cq, CQEventSelNextWC); + rc = iba_rearm_cq(kibnal_data.kib_cq, CQEventSelNextWC); if (rc != 0) { CERROR ("Failed to re-arm completion queue: %d\n", rc); goto failed; diff --git a/lnet/klnds/iiblnd/iiblnd.h b/lnet/klnds/iiblnd/iiblnd.h index c081417..ed9d79b 100644 --- a/lnet/klnds/iiblnd/iiblnd.h +++ b/lnet/klnds/iiblnd/iiblnd.h @@ -214,11 +214,9 @@ typedef struct IB_HANDLE kib_cq; /* completion queue */ kib_md_t kib_whole_mem; /* whole-mem registration */ - IBT_INTERFACE_UNION kib_interfaces; /* The Infinicon IBT interface */ - - int kib_hca_idx; /* my HCA number */ - uint64 kib_hca_guids[8]; /* all the HCA guids */ - IB_CA_ATTRIBUTES kib_hca_attrs; /* where to get HCA attrs */ + int kib_hca_idx; /* my HCA number */ + uint64 kib_hca_guids[8]; /* all the HCA guids */ + IB_CA_ATTRIBUTES kib_hca_attrs; /* where to get HCA attrs */ COMMAND_CONTROL_PARAMETERS kib_sdretry; /* control SD query retries */ } kib_data_t; @@ -445,277 +443,6 @@ extern kib_data_t kibnal_data; extern kib_tunables_t kibnal_tunables; /******************************************************************************/ -/* Infinicon IBT interface wrappers */ -#define IIBT_IF (kibnal_data.kib_interfaces.ver2) - -static inline FSTATUS -iibt_get_hca_guids(uint32 *hca_count, EUI64 *hca_guid_list) -{ - return IIBT_IF.GetCaGuids(hca_count, hca_guid_list); -} - -static inline FSTATUS -iibt_open_hca(EUI64 hca_guid, - IB_COMPLETION_CALLBACK completion_callback, - IB_ASYNC_EVENT_CALLBACK async_event_callback, - void *arg, - IB_HANDLE *handle) -{ - return IIBT_IF.Vpi.OpenCA(hca_guid, completion_callback, - async_event_callback, arg, handle); -} - -static inline FSTATUS -iibt_query_hca(IB_HANDLE hca_handle, IB_CA_ATTRIBUTES *attrs, void **argp) -{ - return IIBT_IF.Vpi.QueryCA(hca_handle, attrs, argp); -} - -static inline FSTATUS -iibt_close_hca(IB_HANDLE hca_handle) -{ - return IIBT_IF.Vpi.CloseCA(hca_handle); -} - -static inline FSTATUS -iibt_pd_allocate(IB_HANDLE hca_handle, __u32 max_avs, IB_HANDLE *pd_handle) -{ - return IIBT_IF.Vpi.AllocatePD(hca_handle, max_avs, pd_handle); -} - -static inline FSTATUS -iibt_pd_free(IB_HANDLE pd_handle) -{ - return IIBT_IF.Vpi.FreePD(pd_handle); -} - -static inline FSTATUS -iibt_register_physical_memory(IB_HANDLE hca_handle, - IB_VIRT_ADDR requested_io_va, - void *phys_buffers, uint64 nphys_buffers, - uint32 io_va_offset, IB_HANDLE pd_handle, - IB_ACCESS_CONTROL access, - IB_HANDLE *mem_handle, - IB_VIRT_ADDR *actual_io_va, - IB_L_KEY *lkey, IB_R_KEY *rkey) -{ - return IIBT_IF.Vpi.RegisterPhysMemRegion(hca_handle, requested_io_va, - phys_buffers, nphys_buffers, - io_va_offset, pd_handle, - access, - mem_handle, actual_io_va, - lkey, rkey); -} - -static inline FSTATUS -iibt_register_contig_physical_memory(IB_HANDLE hca_handle, - IB_VIRT_ADDR requested_io_va, - IB_MR_PHYS_BUFFER *phys_buffers, - uint64 nphys_buffers, - uint32 io_va_offset, IB_HANDLE pd_handle, - IB_ACCESS_CONTROL access, - IB_HANDLE *mem_handle, - IB_VIRT_ADDR *actual_io_va, - IB_L_KEY *lkey, IB_R_KEY *rkey) -{ - return IIBT_IF.Vpi.RegisterContigPhysMemRegion(hca_handle, - requested_io_va, - phys_buffers, - nphys_buffers, - io_va_offset, pd_handle, - access, - mem_handle, actual_io_va, - lkey, rkey); -} - -static inline FSTATUS -iibt_register_memory(IB_HANDLE hca_handle, - void *virt_addr, unsigned int length, - IB_HANDLE pd_handle, - IB_ACCESS_CONTROL access, - IB_HANDLE *mem_handle, - IB_L_KEY *lkey, IB_R_KEY *rkey) -{ - return IIBT_IF.Vpi.RegisterMemRegion(hca_handle, - virt_addr, length, - pd_handle, - access, - mem_handle, - lkey, rkey); -} - -static inline FSTATUS -iibt_deregister_memory(IB_HANDLE mem_handle) -{ - return IIBT_IF.Vpi.DeregisterMemRegion(mem_handle); -} - -static inline FSTATUS -iibt_cq_create(IB_HANDLE hca_handle, uint32 requested_size, - void *arg, IB_HANDLE *cq_handle, uint32 *actual_size) -{ - return IIBT_IF.Vpi.CreateCQ(hca_handle, requested_size, - arg, cq_handle, actual_size); -} - -static inline FSTATUS -iibt_cq_poll(IB_HANDLE cq_handle, IB_WORK_COMPLETION *wc) -{ - return IIBT_IF.Vpi.PollCQ(cq_handle, wc); -} - -static inline FSTATUS -iibt_cq_rearm(IB_HANDLE cq_handle, IB_CQ_EVENT_SELECT select) -{ - return IIBT_IF.Vpi.RearmCQ(cq_handle, select); -} - -static inline FSTATUS -iibt_cq_destroy(IB_HANDLE cq_handle) -{ - return IIBT_IF.Vpi.DestroyCQ(cq_handle); -} - -static inline FSTATUS -iibt_qp_create(IB_HANDLE hca_handle, IB_QP_ATTRIBUTES_CREATE *create_attr, - void *arg, IB_HANDLE *cq_handle, - IB_QP_ATTRIBUTES_QUERY *query_attr) -{ - return IIBT_IF.Vpi.CreateQP(hca_handle, create_attr, arg, cq_handle, - query_attr); -} - -static inline FSTATUS -iibt_qp_query(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_QUERY *query_attr, - void **arg_ptr) -{ - return IIBT_IF.Vpi.QueryQP(qp_handle, query_attr, arg_ptr); -} - -static inline FSTATUS -iibt_qp_modify(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_MODIFY *modify_attr, - IB_QP_ATTRIBUTES_QUERY *query_attr) -{ - return IIBT_IF.Vpi.ModifyQP(qp_handle, modify_attr, query_attr); -} - -static inline FSTATUS -iibt_qp_destroy(IB_HANDLE qp_handle) -{ - return IIBT_IF.Vpi.DestroyQP(qp_handle); -} - - -static inline FSTATUS -iibt_postrecv2(IB_HANDLE qp_handle, IB_WORK_REQ2 *work_req, - IB_WORK_REQ2 **failed_work_req) -{ - return IIBT_IF.Vpi.PostRecv2(qp_handle, work_req, failed_work_req); -} - -static inline FSTATUS -iibt_postsend2(IB_HANDLE qp_handle, IB_WORK_REQ2 *work_req, - IB_WORK_REQ2 **failed_work_req) -{ - return IIBT_IF.Vpi.PostSend2(qp_handle, work_req, failed_work_req); -} - -static inline FSTATUS -iibt_sd_register(IB_HANDLE *sd_handle, CLIENT_CONTROL_PARAMETERS *p) -{ - return IIBT_IF.Sdi.Register(sd_handle, p); -} - -static inline FSTATUS -iibt_sd_deregister(IB_HANDLE sd_handle) -{ - return IIBT_IF.Sdi.Deregister(sd_handle); -} - -static inline FSTATUS -iibt_sd_port_fabric_operation(IB_HANDLE sd_handle, EUI64 port_guid, - FABRIC_OPERATION_DATA *fod, - PFABRIC_OPERATION_CALLBACK callback, - COMMAND_CONTROL_PARAMETERS *p, void *arg) -{ - return IIBT_IF.Sdi.PortFabricOperation(sd_handle, port_guid, - fod, callback, p, arg); -} - -static inline FSTATUS -iibt_sd_query_port_fabric_information(IB_HANDLE sd_handle, EUI64 port_guid, - QUERY *qry, - PQUERY_CALLBACK callback, - COMMAND_CONTROL_PARAMETERS *p, void *arg) -{ - return IIBT_IF.Sdi.QueryPortFabricInformation(sd_handle, port_guid, - qry, callback, p, arg); -} - -static inline IB_HANDLE -iibt_cm_create_cep(CM_CEP_TYPE type) -{ - return IIBT_IF.Cmi.CmCreateCEP(type); -} - -static inline FSTATUS -iibt_cm_modify_cep(IB_HANDLE cep, uint32 attr, char* value, uint32 len, - uint32 offset) -{ - return IIBT_IF.Cmi.CmModifyCEP(cep, attr, value, len, offset); -} - -static inline FSTATUS -iibt_cm_destroy_cep(IB_HANDLE cep_handle) -{ - return IIBT_IF.Cmi.CmDestroyCEP(cep_handle); -} - -static inline FSTATUS -iibt_cm_listen(IB_HANDLE cep, CM_LISTEN_INFO *info, - PFN_CM_CALLBACK callback, void *arg) -{ - return IIBT_IF.Cmi.CmListen(cep, info, callback, arg); -} - -static inline FSTATUS -iibt_cm_cancel(IB_HANDLE cep) -{ - return IIBT_IF.Cmi.CmCancel(cep); -} - -static inline FSTATUS -iibt_cm_accept(IB_HANDLE cep, - CM_CONN_INFO *send_info, CM_CONN_INFO *recv_info, - PFN_CM_CALLBACK callback, void *arg, - IB_HANDLE *new_cep) -{ - return IIBT_IF.Cmi.CmAccept(cep, - send_info, recv_info, - callback, arg, new_cep); -} - -static inline FSTATUS -iibt_cm_reject(IB_HANDLE cep, CM_REJECT_INFO *rej) -{ - return IIBT_IF.Cmi.CmReject(cep, rej); -} - -static inline FSTATUS -iibt_cm_disconnect(IB_HANDLE cep, CM_DREQUEST_INFO *req, - CM_DREPLY_INFO *reply) -{ - return IIBT_IF.Cmi.CmDisconnect(cep, req, reply); -} - -static inline FSTATUS -iibt_cm_connect (IB_HANDLE cep, CM_REQUEST_INFO *req, - PFN_CM_CALLBACK callback, void *arg) -{ - return IIBT_IF.Cmi.CmConnect (cep, req, callback, arg); -} - -/******************************************************************************/ /* these are purposely avoiding using local vars so they don't increase * stack consumption. */ @@ -889,22 +616,13 @@ kibnal_wreqid2type (__u64 wreqid) return (wreqid & IBNAL_WID_MASK); } -#if 0 static inline void kibnal_set_conn_state (kib_conn_t *conn, int state) { - CDEBUG(D_WARNING,"%p state %d\n", conn, state); + CDEBUG(D_NET,"%p state %d\n", conn, state); conn->ibc_state = state; mb(); } -#else -#define kibnal_set_conn_state(conn, state) \ -do { \ - CDEBUG(D_WARNING,"%p state %d\n", conn, state); \ - conn->ibc_state = state; \ - mb(); \ -} while (0) -#endif #if IBNAL_USE_FMR diff --git a/lnet/klnds/iiblnd/iiblnd_cb.c b/lnet/klnds/iiblnd/iiblnd_cb.c index c486116..3f934d1 100644 --- a/lnet/klnds/iiblnd/iiblnd_cb.c +++ b/lnet/klnds/iiblnd/iiblnd_cb.c @@ -141,7 +141,7 @@ kibnal_post_rx (kib_rx_t *rx, int credit) }; rx->rx_wrq = (IB_WORK_REQ2) { - .Next = NULL; + .Next = NULL, .WorkReqId = kibnal_ptr2wreqid(rx, IBNAL_WID_RX), .MessageLen = IBNAL_MSG_SIZE, .DSList = &rx->rx_gl, @@ -166,7 +166,7 @@ kibnal_post_rx (kib_rx_t *rx, int credit) rx->rx_nob = -1; /* flag posted */ mb(); - frc = iibt_postrecv2(conn->ibc_qp, &rx->rx_wrq, NULL); + frc = iba_post_recv2(conn->ibc_qp, &rx->rx_wrq, NULL); if (frc == FSUCCESS) { if (credit) { spin_lock(&conn->ibc_lock); @@ -801,7 +801,6 @@ kibnal_check_sends (kib_conn_t *conn) FSTATUS frc; int rc; int done; - int i; LASSERT (conn->ibc_state >= IBNAL_CONN_ESTABLISHED); @@ -897,7 +896,7 @@ kibnal_check_sends (kib_conn_t *conn) if (conn->ibc_state != IBNAL_CONN_ESTABLISHED) { rc = -ECONNABORTED; } else { - frc = iibt_postsend2(conn->ibc_qp, tx->tx_wrq, NULL); + frc = iba_post_send2(conn->ibc_qp, tx->tx_wrq, NULL); if (frc != FSUCCESS) rc = -EIO; } @@ -1037,7 +1036,7 @@ kibnal_init_rdma (kib_tx_t *tx, int type, int nob, kib_msg_t *ibmsg = tx->tx_msg; kib_rdma_desc_t *srcrd = tx->tx_rd; IB_LOCAL_DATASEGMENT *gl; - IB_WORK_REQ *wrq; + IB_WORK_REQ2 *wrq; int rc; #if IBNAL_USE_FMR @@ -1798,7 +1797,7 @@ kibnal_conn_disconnected(kib_conn_t *conn) kibnal_set_conn_state(conn, IBNAL_CONN_DISCONNECTED); /* move QP to error state to make posted work items complete */ - frc = iibt_qp_modify(conn->ibc_qp, &qpam, NULL); + frc = iba_modify_qp(conn->ibc_qp, &qpam, NULL); if (frc != FSUCCESS) CERROR("can't move qp state to error: %d\n", frc); @@ -1945,7 +1944,7 @@ kibnal_connreq_done (kib_conn_t *conn, int active, int status) LASSERT(conn->ibc_state == IBNAL_CONN_CONNECTING); kibnal_set_conn_state(conn, IBNAL_CONN_ESTABLISHED); - CDEBUG(D_WARNING, "Connection %s ESTABLISHED\n", + CDEBUG(D_NET, "Connection %s ESTABLISHED\n", libcfs_nid2str(conn->ibc_peer->ibp_nid)); write_lock_irqsave(&kibnal_data.kib_global_lock, flags); @@ -2003,7 +2002,7 @@ kibnal_reject (lnet_nid_t nid, IB_HANDLE cep, int reason) LASSERT (msg < &msgs[nmsg]); - frc = iibt_cm_reject(cep, msg); + frc = iba_cm_reject(cep, msg); if (frc != FSUCCESS) CERROR("Error %d rejecting %s\n", frc, libcfs_nid2str(nid)); } @@ -2047,7 +2046,7 @@ kibnal_check_connreject(kib_conn_t *conn, int active, CM_REJECT_INFO *rej) void kibnal_cm_disconnect_callback(kib_conn_t *conn, CM_CONN_INFO *info) { - CDEBUG(D_WARNING, "%s: state %d, status 0x%x\n", + CDEBUG(D_NET, "%s: state %d, status 0x%x\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), conn->ibc_state, info->Status); @@ -2059,7 +2058,7 @@ kibnal_cm_disconnect_callback(kib_conn_t *conn, CM_CONN_INFO *info) break; case FCM_DISCONNECT_REQUEST: - /* Schedule conn to iibt_cm_disconnect() if it wasn't already */ + /* Schedule conn to iba_cm_disconnect() if it wasn't already */ kibnal_close_conn (conn, 0); break; @@ -2255,16 +2254,16 @@ kibnal_listen_callback(IB_HANDLE cep, CM_CONN_INFO *info, void *arg) LASSERT (conn->ibc_cep == NULL); kibnal_set_conn_state(conn, IBNAL_CONN_CONNECTING); - frc = iibt_cm_accept(cep, - &conn->ibc_cvars->cv_cmci, - NULL, - kibnal_cm_passive_callback, conn, - &conn->ibc_cep); + frc = iba_cm_accept(cep, + &conn->ibc_cvars->cv_cmci, + NULL, + kibnal_cm_passive_callback, conn, + &conn->ibc_cep); if (frc == FSUCCESS || frc == FPENDING) return; - CERROR("iibt_cm_accept(%s) failed: %d\n", + CERROR("iba_cm_accept(%s) failed: %d\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), frc); kibnal_connreq_done(conn, 0, -ECONNABORTED); } @@ -2344,9 +2343,9 @@ kibnal_check_connreply(kib_conn_t *conn, CM_REPLY_INFO *rep) memset(&conn->ibc_cvars->cv_cmci, 0, sizeof(conn->ibc_cvars->cv_cmci)); - frc = iibt_cm_accept(conn->ibc_cep, - &conn->ibc_cvars->cv_cmci, - NULL, NULL, NULL, NULL); + frc = iba_cm_accept(conn->ibc_cep, + &conn->ibc_cvars->cv_cmci, + NULL, NULL, NULL, NULL); if (frc == FCM_CONNECT_ESTABLISHED) { kibnal_connreq_done(conn, 1, 0); @@ -2476,8 +2475,8 @@ kibnal_pathreq_callback (void *arg, QUERY *qry, kibnal_set_conn_state(conn, IBNAL_CONN_CONNECTING); /* cm callback gets my conn ref */ - frc = iibt_cm_connect(conn->ibc_cep, req, - kibnal_cm_active_callback, conn); + frc = iba_cm_connect(conn->ibc_cep, req, + kibnal_cm_active_callback, conn); if (frc == FPENDING || frc == FSUCCESS) return; @@ -2543,12 +2542,12 @@ kibnal_service_get_callback (void *arg, QUERY *qry, conn->ibc_cvars->cv_svcrec.RID.ServiceGID.Type.Global.InterfaceID; /* kibnal_pathreq_callback gets my conn ref */ - frc = iibt_sd_query_port_fabric_information(kibnal_data.kib_sd, - kibnal_data.kib_port_guid, - qry, - kibnal_pathreq_callback, - &kibnal_data.kib_sdretry, - conn); + frc = iba_sd_query_port_fabric_info(kibnal_data.kib_sd, + kibnal_data.kib_port_guid, + qry, + kibnal_pathreq_callback, + &kibnal_data.kib_sdretry, + conn); if (frc == FPENDING) return; @@ -2589,12 +2588,12 @@ kibnal_connect_peer (kib_peer_t *peer) peer->ibp_nid); /* kibnal_service_get_callback gets my conn ref */ - frc = iibt_sd_query_port_fabric_information(kibnal_data.kib_sd, - kibnal_data.kib_port_guid, - qry, - kibnal_service_get_callback, - &kibnal_data.kib_sdretry, - conn); + frc = iba_sd_query_port_fabric_info(kibnal_data.kib_sd, + kibnal_data.kib_port_guid, + qry, + kibnal_service_get_callback, + &kibnal_data.kib_sdretry, + conn); if (frc == FPENDING) return; @@ -2702,7 +2701,7 @@ kibnal_disconnect_conn (kib_conn_t *conn) kibnal_conn_disconnected(conn); - frc = iibt_cm_disconnect(conn->ibc_cep, NULL, NULL); + frc = iba_cm_disconnect(conn->ibc_cep, NULL, NULL); switch (frc) { case FSUCCESS: break; @@ -2902,11 +2901,11 @@ kibnal_scheduler(void *arg) spin_unlock_irqrestore(&kibnal_data.kib_sched_lock, flags); - frc = iibt_cq_poll(kibnal_data.kib_cq, &wc); + frc = iba_poll_cq(kibnal_data.kib_cq, &wc); if (frc == FNOT_DONE) { /* CQ empty */ - frc2 = iibt_cq_rearm(kibnal_data.kib_cq, - CQEventSelNextWC); + frc2 = iba_rearm_cq(kibnal_data.kib_cq, + CQEventSelNextWC); LASSERT (frc2 == FSUCCESS); } -- 1.8.3.1