From 5c9f09d49e4caffe359f99bacd164338a44a3cda Mon Sep 17 00:00:00 2001 From: Sonia Sharma Date: Mon, 9 Jul 2018 21:03:19 -0400 Subject: [PATCH] LU-9667 lnet: Add o2iblnd peer sysfs structure Adding APIs to create kobjects and the attribute files for o2iblnd sysfs directory structure. Callback functions for the o2iblnd peer attributes added in a new file o2iblnd_sysfs.c Test-Parameters: trivial Change-Id: Icd1914194b7eb179ee4cc037900a204e945cb00b Signed-off-by: Sonia Sharma Reviewed-on: https://review.whamcloud.com/29164 Reviewed-by: Amir Shehata Tested-by: Jenkins Reviewed-by: James Simmons Tested-by: Maloo Reviewed-by: Sonia Sharma --- lnet/include/lnet/lnet-sysfs.h | 25 +++++- lnet/klnds/o2iblnd/Makefile.in | 2 +- lnet/klnds/o2iblnd/o2iblnd.c | 10 +++ lnet/klnds/o2iblnd/o2iblnd.h | 4 + lnet/klnds/o2iblnd/o2iblnd_sysfs.c | 151 +++++++++++++++++++++++++++++++++++++ lnet/lnet/api-ni.c | 6 +- lnet/lnet/lnet-sysfs.c | 90 ++++++++++++++++++++++ 7 files changed, 284 insertions(+), 4 deletions(-) create mode 100644 lnet/klnds/o2iblnd/o2iblnd_sysfs.c diff --git a/lnet/include/lnet/lnet-sysfs.h b/lnet/include/lnet/lnet-sysfs.h index cc7f8de..c350c9b 100644 --- a/lnet/include/lnet/lnet-sysfs.h +++ b/lnet/include/lnet/lnet-sysfs.h @@ -49,7 +49,6 @@ #include #include - extern struct kobject *lnet_kobj; /* Sysfs lnet kobject */ struct sysfs_lnd_conn { @@ -157,4 +156,28 @@ int lnet_peer_ni_sysfs_setup(struct lnet_peer_ni *lpni, struct lnet_peer *lp); */ void lnet_peer_ni_sysfs_cleanup(struct lnet_sysfs_peer *spni); +/* + * lnd_peer_sysfs_setup + * + * Setup the sysfs hierarchy for lnd peer stats. + * Create kobject for the peer_nid, kobject for the + * local_nid and the attribute files for the peer_ni. + * + * peer_nid - peer ni NID + * nid - local ni + * peer_ni - LND peer ni for which stats are being created + */ +extern int lnd_peer_sysfs_setup(lnet_nid_t peer_nid, struct lnet_ni *ni, + struct sysfs_lnd_peer *peer_ni); + +/* + * lnd_peer_sysfs_cleanup + * + * Cleanup the sysfs hierarchy for lnd peer stats. + * Remove the kobject for the lnd peer_ni and the + * attribute files for that peer_ni. + * + * peer_ni - LND peer_ni for which stats are being cleaned up + */ +extern void lnd_peer_sysfs_cleanup(struct sysfs_lnd_peer *peer_ni); #endif diff --git a/lnet/klnds/o2iblnd/Makefile.in b/lnet/klnds/o2iblnd/Makefile.in index 34b82fe..98a22bf 100644 --- a/lnet/klnds/o2iblnd/Makefile.in +++ b/lnet/klnds/o2iblnd/Makefile.in @@ -1,5 +1,5 @@ MODULES := ko2iblnd -ko2iblnd-objs := o2iblnd.o o2iblnd_cb.o o2iblnd_modparams.o +ko2iblnd-objs := o2iblnd.o o2iblnd_cb.o o2iblnd_modparams.o o2iblnd_sysfs.o # Need to make sure that an external OFED source pool overrides # any in-kernel OFED sources diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index cb5ebb5..68eee41 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -318,6 +318,7 @@ kiblnd_create_peer(struct lnet_ni *ni, struct kib_peer_ni **peerp, struct kib_peer_ni *peer_ni; struct kib_net *net = ni->ni_data; int cpt = lnet_cpt_of_nid(nid, ni); + int rc = 0; unsigned long flags; LASSERT(net != NULL); @@ -329,6 +330,12 @@ kiblnd_create_peer(struct lnet_ni *ni, struct kib_peer_ni **peerp, return -ENOMEM; } + rc = set_sysfs_peer(ni, nid, &peer_ni->ibp_sysfs); + if (rc) { + LIBCFS_FREE(peer_ni, sizeof(*peer_ni)); + return rc; + } + peer_ni->ibp_ni = ni; peer_ni->ibp_nid = nid; peer_ni->ibp_error = 0; @@ -366,6 +373,9 @@ kiblnd_destroy_peer(struct kib_peer_ni *peer_ni) LASSERT(kiblnd_peer_idle(peer_ni)); LASSERT(list_empty(&peer_ni->ibp_tx_queue)); + /* clean up the o2iblnd sysfs structure */ + lnd_peer_sysfs_cleanup(&peer_ni->ibp_sysfs); + LIBCFS_FREE(peer_ni, sizeof(*peer_ni)); /* NB a peer_ni's connections keep a reference on their peer_ni until diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index dd04ab0..32b0f41 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -77,6 +77,7 @@ #include #include +#include #define IBLND_PEER_HASH_SIZE 101 /* # peer_ni lists */ /* # scheduler loops before reschedule */ @@ -759,6 +760,9 @@ struct kib_peer_ni { struct sysfs_lnd_peer ibp_sysfs; }; +extern int set_sysfs_peer(struct lnet_ni *ni, lnet_nid_t nid, + struct sysfs_lnd_peer *peer); + #ifndef HAVE_IB_INC_RKEY /** * ib_inc_rkey - increments the key portion of the given rkey. Can be used diff --git a/lnet/klnds/o2iblnd/o2iblnd_sysfs.c b/lnet/klnds/o2iblnd/o2iblnd_sysfs.c new file mode 100644 index 0000000..4e6d1b2e8 --- /dev/null +++ b/lnet/klnds/o2iblnd/o2iblnd_sysfs.c @@ -0,0 +1,151 @@ +/* + * LGPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + * LGPL HEADER END + * + * Copyright (c) 2014, 2016, Intel Corporation. + * + * Author: + * Sonia Sharma + */ + +#include "o2iblnd.h" + +/* + * LND peer stats attribute callback functions + */ +static struct kib_peer_ni *get_kib_peer(struct kobject *kobj) +{ + struct sysfs_lnd_peer *peer = container_of(kobj, struct sysfs_lnd_peer, + stats_kobj); + return container_of(peer, struct kib_peer_ni, ibp_sysfs); +} + +static ssize_t ibp_conns_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&peer_ni->ibp_conns)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &peer_ni->ibp_conns) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibp_conns); + +static ssize_t ibp_tx_queue_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + struct list_head *ctmp; + int count = 0; + + if (list_empty(&peer_ni->ibp_tx_queue)) + return snprintf(buf, PAGE_SIZE, "%d\n", count); + + list_for_each(ctmp, &peer_ni->ibp_tx_queue) + count++; + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibp_tx_queue); + +static ssize_t ibp_accepting_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + unsigned short count = peer_ni->ibp_accepting; + + return snprintf(buf, PAGE_SIZE, "%hu\n", count); +} +LNET_RO_ATTR(ibp_accepting); + +static ssize_t ibp_connecting_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + unsigned short count = peer_ni->ibp_connecting; + + return snprintf(buf, PAGE_SIZE, "%hu\n", count); +} +LNET_RO_ATTR(ibp_connecting); + +static ssize_t ibp_races_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + unsigned char count = peer_ni->ibp_races; + + return snprintf(buf, PAGE_SIZE, "%u\n", count); +} +LNET_RO_ATTR(ibp_races); + +static ssize_t ibp_last_alive_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + time64_t count = peer_ni->ibp_last_alive; + + return snprintf(buf, PAGE_SIZE, "%lld\n", count ? count : -1); +} +LNET_RO_ATTR(ibp_last_alive); + +static ssize_t ibp_refcount_show(struct kobject *kobj, struct attribute *attr, + char *buf) { + struct kib_peer_ni *peer_ni = get_kib_peer(kobj); + int count = atomic_read(&peer_ni->ibp_refcount); + + return snprintf(buf, PAGE_SIZE, "%d\n", count); +} +LNET_RO_ATTR(ibp_refcount); + +/* + * LND peer stats attributes + */ +static struct attribute *o2iblnd_peer_attrs[] = { + &lnet_attr_ibp_conns.attr, + &lnet_attr_ibp_tx_queue.attr, + &lnet_attr_ibp_accepting.attr, + &lnet_attr_ibp_connecting.attr, + &lnet_attr_ibp_races.attr, + &lnet_attr_ibp_last_alive.attr, + &lnet_attr_ibp_refcount.attr, + NULL, +}; + +static void lnd_peer_sysfs_release(struct kobject *kobj) +{ + struct sysfs_lnd_peer *peer_ni = container_of(kobj, + struct sysfs_lnd_peer, + stats_kobj); + complete(&peer_ni->stats_kobj_unregister); +} + +static struct kobj_type kib_peer_ni_ktype = { + .sysfs_ops = &lnet_sysfs_ops, + .release = lnd_peer_sysfs_release, + .default_attrs = o2iblnd_peer_attrs, +}; + +int set_sysfs_peer(struct lnet_ni *ni, lnet_nid_t nid, + struct sysfs_lnd_peer *kib_peer) +{ + kib_peer->stats_ktype = &kib_peer_ni_ktype; + + /* setup the sysfs kobjects for o2iblnd */ + return lnd_peer_sysfs_setup(nid, ni, kib_peer); +} diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index a4c7877..fc4ece1 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -856,8 +856,10 @@ EXPORT_SYMBOL(lnet_register_lnd); void lnet_unregister_lnd(struct lnet_lnd *lnd) { - kset_put(lnd->lnd_peers_kset); - kobject_put(lnd->lnd_kobj); + if (lnd->lnd_type != LOLND) { + kset_put(lnd->lnd_peers_kset); + kobject_put(lnd->lnd_kobj); + } mutex_lock(&the_lnet.ln_lnd_mutex); diff --git a/lnet/lnet/lnet-sysfs.c b/lnet/lnet/lnet-sysfs.c index 3e12d2e..c7a492b 100644 --- a/lnet/lnet/lnet-sysfs.c +++ b/lnet/lnet/lnet-sysfs.c @@ -952,3 +952,93 @@ void lnet_peer_ni_sysfs_cleanup(struct lnet_sysfs_peer *spni) LIBCFS_FREE(spni, sizeof(*spni)); } + +/* + * LND Peer sysfs structure APIs + */ + +int lnd_peer_sysfs_setup(lnet_nid_t peer_nid, struct lnet_ni *ni, + struct sysfs_lnd_peer *lnd_peer) +{ + struct kset *peers_kset = ni->ni_net->net_lnd->lnd_peers_kset; + struct kobject *peers_kobj = &peers_kset->kobj; + struct kobject *local_ni_dir; + char *pnid = libcfs_nid2str(peer_nid); + char *local_nid = libcfs_nid2str(ni->ni_nid); + int rc = 0; + + /* Before creating the peer_ni kobject, check if it already exists. + * If it exists then compensate for a ref up due to + * kset_find_obj() */ + lnd_peer->peer_ni_kobj = kset_find_obj(peers_kset, pnid); + if (lnd_peer->peer_ni_kobj) { + kobject_put(lnd_peer->peer_ni_kobj); + return rc; + } + + lnd_peer->peer_ni_kobj = kobject_create_and_add(pnid, peers_kobj); + if (!lnd_peer->peer_ni_kobj) { + CERROR("Cannot add new kobject for %s\n", pnid); + return -ENOMEM; + } + + local_ni_dir = kobject_create_and_add("local_ni", + lnd_peer->peer_ni_kobj); + if (!local_ni_dir) { + CERROR("Cannot create directory 'local_ni'\n"); + rc = -ENOMEM; + goto deref_peer_ni; + } + + lnd_peer->local_ni_kobj = kobject_create_and_add(local_nid, + local_ni_dir); + if (!lnd_peer->local_ni_kobj) { + CERROR("Cannot add new kobject for %s\n", local_nid); + rc = -ENOMEM; + goto deref_local_ni_dir; + } + + init_completion(&lnd_peer->stats_kobj_unregister); + + rc = kobject_init_and_add(&lnd_peer->stats_kobj, lnd_peer->stats_ktype, + lnd_peer->local_ni_kobj, "%s", "stats"); + if (rc) { + CERROR("Cannot add new kobject for stats under %s\n", local_nid); + goto deref_local_ni; + } + + lnd_peer->peer_conns_kobj = kobject_create_and_add("conns", + lnd_peer->local_ni_kobj); + if (!lnd_peer->peer_conns_kobj) { + CERROR("Could not add kobject for conns under %s\n", local_nid); + rc = -ENOMEM; + goto deref_stats; + } + + return rc; + +deref_stats: + kobject_put(&lnd_peer->stats_kobj); +deref_local_ni: + kobject_put(lnd_peer->local_ni_kobj); +deref_local_ni_dir: + kobject_put(local_ni_dir); +deref_peer_ni: + kobject_put(lnd_peer->peer_ni_kobj); + + return rc; +} +EXPORT_SYMBOL(lnd_peer_sysfs_setup); + +void lnd_peer_sysfs_cleanup(struct sysfs_lnd_peer *peer_ni) +{ + struct kobject *local_ni_dir = peer_ni->local_ni_kobj->parent; + + kobject_put(peer_ni->peer_conns_kobj); + kobject_put(&peer_ni->stats_kobj); + wait_for_completion(&peer_ni->stats_kobj_unregister); + kobject_put(peer_ni->local_ni_kobj); + kobject_put(local_ni_dir); + kobject_put(peer_ni->peer_ni_kobj); +} +EXPORT_SYMBOL(lnd_peer_sysfs_cleanup); -- 1.8.3.1