From 12f31fcf14020e3b807e059d70bb965836690919 Mon Sep 17 00:00:00 2001 From: Sonia Sharma Date: Mon, 30 Jul 2018 19:29:28 -0400 Subject: [PATCH] LU-9667 lnet: Add o2iblnd connection sysfs structure Adding APIs to create kobjects and the attribute files for o2iblnd connections sysfs directory structure. Callback functions for the o2iblnd conn attributes are added in o2iblnd_sysfs.c Test-Parameters: trivial Change-Id: I77604757be46af68ecacb4cb425477e70fa4b89b Signed-off-by: Sonia Sharma Reviewed-on: https://review.whamcloud.com/31503 Tested-by: Jenkins Reviewed-by: Amir Shehata Reviewed-by: James Simmons Tested-by: Maloo Reviewed-by: Sonia Sharma --- lnet/include/lnet/lnet-sysfs.h | 26 ++++++ lnet/klnds/o2iblnd/o2iblnd.c | 22 +++-- lnet/klnds/o2iblnd/o2iblnd.h | 4 + lnet/klnds/o2iblnd/o2iblnd_sysfs.c | 180 +++++++++++++++++++++++++++++++++++++ lnet/lnet/lnet-sysfs.c | 47 +++++++++- 5 files changed, 270 insertions(+), 9 deletions(-) diff --git a/lnet/include/lnet/lnet-sysfs.h b/lnet/include/lnet/lnet-sysfs.h index c350c9b..e465e16 100644 --- a/lnet/include/lnet/lnet-sysfs.h +++ b/lnet/include/lnet/lnet-sysfs.h @@ -180,4 +180,30 @@ extern int lnd_peer_sysfs_setup(lnet_nid_t peer_nid, struct lnet_ni *ni, * peer_ni - LND peer_ni for which stats are being cleaned up */ extern void lnd_peer_sysfs_cleanup(struct sysfs_lnd_peer *peer_ni); + +/* + * lnd_conn_sysfs_setup + * + * Setup the sysfs hierarchy for lnd connection stats. + * Create kobject for the conn_id, kobject for the + * stats under conn_id and the attribute files for the + * connection. + * + * peer_ni - LND peer ni under which the conn is created + * lnd_conn - LND conn for which attribute files are created + */ +extern int lnd_conn_sysfs_setup(struct sysfs_lnd_peer *peer_ni, + struct sysfs_lnd_conn *lnd_conn); + +/* + * lnd_conn_sysfs_cleanup + * + * Cleanup the sysfs hierarchy for lnd peer stats. + * Remove the kobject for the lnd peer_ni and the + * attribute files for that peer_ni. + * + * conn - LND conn for which stats are being cleaned up + * + */ +extern void lnd_conn_sysfs_cleanup(struct sysfs_lnd_conn *conn); #endif diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 68eee41..03201a8 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -835,18 +835,22 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, INIT_LIST_HEAD(&conn->ibc_zombie_txs); spin_lock_init(&conn->ibc_lock); + rc = set_sysfs_conn(&peer_ni->ibp_sysfs, &conn->ibc_sysfs); + if (rc) + goto failed_2; + LIBCFS_CPT_ALLOC(conn->ibc_connvars, lnet_cpt_table(), cpt, sizeof(*conn->ibc_connvars)); if (conn->ibc_connvars == NULL) { CERROR("Can't allocate in-progress connection state\n"); - goto failed_2; + goto sysfs_cleanup; } write_lock_irqsave(glock, flags); if (dev->ibd_failover) { write_unlock_irqrestore(glock, flags); CERROR("%s: failover in progress\n", dev->ibd_ifname); - goto failed_2; + goto sysfs_cleanup; } if (dev->ibd_hdev->ibh_ibdev != cmid->device) { @@ -860,7 +864,7 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, write_unlock_irqrestore(glock, flags); CERROR("cmid HCA(%s), kib_dev(%s) need failover\n", cmid->device->name, dev->ibd_ifname); - goto failed_2; + goto sysfs_cleanup; } kiblnd_hdev_addref_locked(dev->ibd_hdev); @@ -891,7 +895,7 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, */ CERROR("Failed to create CQ with %d CQEs: %ld\n", IBLND_CQ_ENTRIES(conn), PTR_ERR(cq)); - goto failed_2; + goto sysfs_cleanup; } conn->ibc_cq = cq; @@ -899,7 +903,7 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); if (rc != 0) { CERROR("Can't request completion notification: %d\n", rc); - goto failed_2; + goto sysfs_cleanup; } init_qp_attr->event_handler = kiblnd_qp_event; @@ -931,7 +935,7 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, init_qp_attr->cap.max_recv_wr, init_qp_attr->cap.max_send_sge, init_qp_attr->cap.max_recv_sge); - goto failed_2; + goto sysfs_cleanup; } if (conn->ibc_queue_depth != peer_ni->ibp_queue_depth) @@ -945,13 +949,13 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, IBLND_RX_MSGS(conn) * sizeof(struct kib_rx)); if (conn->ibc_rxs == NULL) { CERROR("Cannot allocate RX buffers\n"); - goto failed_2; + goto sysfs_cleanup; } rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt, IBLND_RX_MSG_PAGES(conn)); if (rc != 0) - goto failed_2; + goto sysfs_cleanup; kiblnd_map_rx_descs(conn); @@ -999,6 +1003,8 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid, atomic_inc(&net->ibn_nconns); return conn; + sysfs_cleanup: + lnd_conn_sysfs_cleanup(&conn->ibc_sysfs); failed_2: kiblnd_destroy_conn(conn); LIBCFS_FREE(conn, sizeof(*conn)); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 32b0f41..923ff83 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -712,6 +712,9 @@ struct kib_conn { struct sysfs_lnd_conn ibc_sysfs; }; +extern int set_sysfs_conn(struct sysfs_lnd_peer *kib_peer, + struct sysfs_lnd_conn *conn); + #define IBLND_CONN_INIT 0 /* being initialised */ #define IBLND_CONN_ACTIVE_CONNECT 1 /* active sending req */ #define IBLND_CONN_PASSIVE_WAIT 2 /* passive waiting for rtu */ @@ -847,6 +850,7 @@ do { \ (conn), atomic_read(&(conn)->ibc_refcount)); \ LASSERT_ATOMIC_POS(&(conn)->ibc_refcount); \ if (atomic_dec_and_test(&(conn)->ibc_refcount)) { \ + lnd_conn_sysfs_cleanup(&conn->ibc_sysfs); \ spin_lock_irqsave(&kiblnd_data.kib_connd_lock, flags); \ list_add_tail(&(conn)->ibc_list, \ &kiblnd_data.kib_connd_zombies); \ diff --git a/lnet/klnds/o2iblnd/o2iblnd_sysfs.c b/lnet/klnds/o2iblnd/o2iblnd_sysfs.c index 4e6d1b2e8..c9ed3cb 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_sysfs.c +++ b/lnet/klnds/o2iblnd/o2iblnd_sysfs.c @@ -114,6 +114,145 @@ static ssize_t ibp_refcount_show(struct kobject *kobj, struct attribute *attr, LNET_RO_ATTR(ibp_refcount); /* + * LND connection stats attribute callback functions + */ +static struct kib_conn *get_kib_conn(struct kobject *kobj) +{ + struct sysfs_lnd_conn *conn = container_of(kobj, struct sysfs_lnd_conn, + stats_kobj); + return container_of(conn, struct kib_conn, ibc_sysfs); +} + +static ssize_t ibc_refcount_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + int count = atomic_read(&conn->ibc_refcount); + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibc_refcount); + +static ssize_t ibc_state_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_state); +} +LNET_RO_ATTR(ibc_state); + +static ssize_t ibc_nsends_posted_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_nsends_posted); +} +LNET_RO_ATTR(ibc_nsends_posted); + +static ssize_t ibc_noops_posted_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_noops_posted); +} +LNET_RO_ATTR(ibc_noops_posted); + +static ssize_t ibc_credits_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_credits); +} +LNET_RO_ATTR(ibc_credits); + +static ssize_t ibc_outstanding_credits_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_outstanding_credits); +} +LNET_RO_ATTR(ibc_outstanding_credits); + +static ssize_t ibc_reserved_credits_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_reserved_credits); +} +LNET_RO_ATTR(ibc_reserved_credits); + +static ssize_t ibc_comms_error_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", conn->ibc_comms_error); +} +LNET_RO_ATTR(ibc_comms_error); + +static ssize_t ibc_tx_queue_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&conn->ibc_tx_queue)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &conn->ibc_tx_queue) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibc_tx_queue); + +static ssize_t ibc_tx_queue_nocred_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&conn->ibc_tx_queue_nocred)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &conn->ibc_tx_queue_nocred) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibc_tx_queue_nocred); + +static ssize_t ibc_tx_queue_rsrvd_show(struct kobject *kobj, + struct attribute *attr, char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&conn->ibc_tx_queue_rsrvd)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &conn->ibc_tx_queue_rsrvd) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibc_tx_queue_rsrvd); + +static ssize_t ibc_active_txs_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_conn *conn = get_kib_conn(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&conn->ibc_active_txs)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &conn->ibc_active_txs) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibc_active_txs); + +/* * LND peer stats attributes */ static struct attribute *o2iblnd_peer_attrs[] = { @@ -149,3 +288,44 @@ int set_sysfs_peer(struct lnet_ni *ni, lnet_nid_t nid, /* setup the sysfs kobjects for o2iblnd */ return lnd_peer_sysfs_setup(nid, ni, kib_peer); } + +/* + * LND conn stats attributes + */ +static struct attribute *o2iblnd_conn_attrs[] = { + &lnet_attr_ibc_refcount.attr, + &lnet_attr_ibc_state.attr, + &lnet_attr_ibc_nsends_posted.attr, + &lnet_attr_ibc_noops_posted.attr, + &lnet_attr_ibc_credits.attr, + &lnet_attr_ibc_outstanding_credits.attr, + &lnet_attr_ibc_reserved_credits.attr, + &lnet_attr_ibc_comms_error.attr, + &lnet_attr_ibc_tx_queue.attr, + &lnet_attr_ibc_tx_queue_nocred.attr, + &lnet_attr_ibc_tx_queue_rsrvd.attr, + &lnet_attr_ibc_active_txs.attr, + NULL, +}; + +static void lnd_conn_sysfs_release(struct kobject *kobj) +{ + struct sysfs_lnd_conn *conn = container_of(kobj, struct sysfs_lnd_conn, + stats_kobj); + complete(&conn->stats_kobj_unregister); +} + +static struct kobj_type kib_conn_ktype = { + .sysfs_ops = &lnet_sysfs_ops, + .release = lnd_conn_sysfs_release, + .default_attrs = o2iblnd_conn_attrs, +}; + +int set_sysfs_conn(struct sysfs_lnd_peer *kib_peer, + struct sysfs_lnd_conn *kib_conn) +{ + kib_conn->stats_ktype = &kib_conn_ktype; + + /* setup the lnd conn sysfs kobjects */ + return lnd_conn_sysfs_setup(kib_peer, kib_conn); +} diff --git a/lnet/lnet/lnet-sysfs.c b/lnet/lnet/lnet-sysfs.c index c7a492b..39bf2e4 100644 --- a/lnet/lnet/lnet-sysfs.c +++ b/lnet/lnet/lnet-sysfs.c @@ -29,7 +29,6 @@ * Sonia Sharma */ - #include #include @@ -1042,3 +1041,49 @@ void lnd_peer_sysfs_cleanup(struct sysfs_lnd_peer *peer_ni) kobject_put(peer_ni->peer_ni_kobj); } EXPORT_SYMBOL(lnd_peer_sysfs_cleanup); + +/* + * LND Connection sysfs setup/cleanup APIs + */ + +int lnd_conn_sysfs_setup(struct sysfs_lnd_peer *peer_ni, + struct sysfs_lnd_conn *lnd_conn) +{ + struct kobject *conn_id_kobj; + char conn_id[LNET_MAX_STR_LEN]; + int rc = 0; + s64 timens = ktime_get_ns(); + + snprintf(conn_id, LNET_MAX_STR_LEN, "%lld", timens); + conn_id_kobj = kobject_create_and_add(conn_id, + peer_ni->peer_conns_kobj); + if (!conn_id_kobj) { + CERROR("Cannot create koject for the connection of %s", + peer_ni->peer_ni_kobj->name); + return rc; + } + + init_completion(&lnd_conn->stats_kobj_unregister); + + rc = kobject_init_and_add(&lnd_conn->stats_kobj, lnd_conn->stats_ktype, + conn_id_kobj, "%s", "stats"); + if (rc) { + CERROR("Cannot create koject for stats under connection %s", + conn_id); + kobject_put(conn_id_kobj); + return rc; + } + + return rc; +} +EXPORT_SYMBOL(lnd_conn_sysfs_setup); + +void lnd_conn_sysfs_cleanup(struct sysfs_lnd_conn *lnd_conn) +{ + struct kobject *conn_id_kobj = lnd_conn->stats_kobj.parent; + + kobject_put(&lnd_conn->stats_kobj); + wait_for_completion(&lnd_conn->stats_kobj_unregister); + kobject_put(conn_id_kobj); +} +EXPORT_SYMBOL(lnd_conn_sysfs_cleanup); -- 1.8.3.1