static void
-lnet_try_destroy_peer_hierarchy_locked(struct lnet_peer_ni *lpni)
+lnet_peer_detach_peer_ni(struct lnet_peer_ni *lpni)
{
- struct lnet_peer_net *peer_net;
- struct lnet_peer *peer;
+ struct lnet_peer_net *lpn;
+ struct lnet_peer *lp;
/* TODO: could the below situation happen? accessing an already
* destroyed peer? */
lpni->lpni_peer_net->lpn_peer == NULL)
return;
- peer_net = lpni->lpni_peer_net;
- peer = lpni->lpni_peer_net->lpn_peer;
+ lpn = lpni->lpni_peer_net;
+ lp = lpni->lpni_peer_net->lpn_peer;
+
+ CDEBUG(D_NET, "peer %s NID %s\n",
+ libcfs_nid2str(lp->lp_primary_nid),
+ libcfs_nid2str(lpni->lpni_nid));
list_del_init(&lpni->lpni_on_peer_net_list);
lpni->lpni_peer_net = NULL;
- /* if peer_net is empty, then remove it from the peer */
- if (list_empty(&peer_net->lpn_peer_nis)) {
- list_del_init(&peer_net->lpn_on_peer_list);
- peer_net->lpn_peer = NULL;
- LIBCFS_FREE(peer_net, sizeof(*peer_net));
+ /* if lpn is empty, then remove it from the peer */
+ if (list_empty(&lpn->lpn_peer_nis)) {
+ list_del_init(&lpn->lpn_on_peer_list);
+ lpn->lpn_peer = NULL;
+ LIBCFS_FREE(lpn, sizeof(*lpn));
/* if the peer is empty then remove it from the
* the_lnet.ln_peers */
- if (list_empty(&peer->lp_peer_nets)) {
- list_del_init(&peer->lp_on_lnet_peer_list);
- LIBCFS_FREE(peer, sizeof(*peer));
+ if (list_empty(&lp->lp_peer_nets)) {
+ list_del_init(&lp->lp_on_lnet_peer_list);
+ LIBCFS_FREE(lp, sizeof(*lp));
}
}
}
ptable->pt_zombies++;
spin_unlock(&ptable->pt_zombie_lock);
- /* no need to keep this peer on the hierarchy anymore */
- lnet_try_destroy_peer_hierarchy_locked(lpni);
+ /* no need to keep this peer_ni on the hierarchy anymore */
+ lnet_peer_detach_peer_ni(lpni);
- /* decrement reference on peer */
+ /* decrement reference on peer_ni */
lnet_peer_ni_decref_locked(lpni);
return 0;
struct lnet_peer_ni *lpni = NULL, *lpni2;
int rc = 0, rc2 = 0;
+ CDEBUG(D_NET, "peer %s\n", libcfs_nid2str(peer->lp_primary_nid));
+
lpni = lnet_get_next_peer_ni_locked(peer, NULL, lpni);
while (lpni != NULL) {
lpni2 = lnet_get_next_peer_ni_locked(peer, NULL, lpni);
}
/*
- * Delete a NID from a peer.
- * Implements a few sanity checks.
- * Call with ln_api_mutex held.
+ * Delete a NID from a peer. Call with ln_api_mutex held.
+ *
+ * Error codes:
+ * -EPERM: Non-DLC deletion from DLC-configured peer.
+ * -ENOENT: No lnet_peer_ni corresponding to the nid.
+ * -ECHILD: The lnet_peer_ni isn't connected to the peer.
+ * -EBUSY: The lnet_peer_ni is the primary, and not the only peer_ni.
*/
static int
-lnet_peer_del_nid(struct lnet_peer *lp, lnet_nid_t nid)
+lnet_peer_del_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags)
{
- struct lnet_peer *lp2;
struct lnet_peer_ni *lpni;
+ lnet_nid_t primary_nid = lp->lp_primary_nid;
+ int rc = 0;
+ if (!(flags & LNET_PEER_CONFIGURED)) {
+ if (lp->lp_state & LNET_PEER_CONFIGURED) {
+ rc = -EPERM;
+ goto out;
+ }
+ }
lpni = lnet_find_peer_ni_locked(nid);
if (!lpni) {
- CERROR("Cannot remove unknown nid %s from peer %s\n",
- libcfs_nid2str(nid),
- libcfs_nid2str(lp->lp_primary_nid));
- return -ENOENT;
+ rc = -ENOENT;
+ goto out;
}
lnet_peer_ni_decref_locked(lpni);
- lp2 = lpni->lpni_peer_net->lpn_peer;
- if (lp2 != lp) {
- CERROR("Nid %s is attached to peer %s, not peer %s\n",
- libcfs_nid2str(nid),
- libcfs_nid2str(lp2->lp_primary_nid),
- libcfs_nid2str(lp->lp_primary_nid));
- return -EINVAL;
+ if (lp != lpni->lpni_peer_net->lpn_peer) {
+ rc = -ECHILD;
+ goto out;
}
/*
* is the only NID.
*/
if (nid == lp->lp_primary_nid && lnet_get_num_peer_nis(lp) != 1) {
- CERROR("Cannot delete primary NID %s from multi-NID peer\n",
- libcfs_nid2str(nid));
- return -EINVAL;
+ rc = -EBUSY;
+ goto out;
}
lnet_net_lock(LNET_LOCK_EX);
lnet_peer_ni_del_locked(lpni);
lnet_net_unlock(LNET_LOCK_EX);
- return 0;
+out:
+ CDEBUG(D_NET, "peer %s NID %s flags %#x: %d\n",
+ libcfs_nid2str(primary_nid), libcfs_nid2str(nid), flags, rc);
+
+ return rc;
}
static void
return NULL;
}
+/*
+ * Always returns 0, but it the last function called from functions
+ * that do return an int, so returning 0 here allows the compiler to
+ * do a tail call.
+ */
static int
-lnet_peer_setup_hierarchy(struct lnet_peer *lp, struct lnet_peer_ni *lpni,
- lnet_nid_t nid)
+lnet_peer_attach_peer_ni(struct lnet_peer *lp,
+ struct lnet_peer_net *lpn,
+ struct lnet_peer_ni *lpni,
+ unsigned flags)
{
- struct lnet_peer_net *lpn = NULL;
struct lnet_peer_table *ptable;
- __u32 net_id = LNET_NIDNET(nid);
-
- /*
- * Create the peer_ni, peer_net, and peer if they don't exist
- * yet.
- */
- if (lp) {
- lpn = lnet_peer_get_net_locked(lp, net_id);
- } else {
- lp = lnet_peer_alloc(nid);
- if (!lp)
- goto out_enomem;
- }
-
- if (!lpn) {
- lpn = lnet_peer_net_alloc(net_id);
- if (!lpn)
- goto out_maybe_free_lp;
- }
-
- if (!lpni) {
- lpni = lnet_peer_ni_alloc(nid);
- if (!lpni)
- goto out_maybe_free_lpn;
- }
/* Install the new peer_ni */
lnet_net_lock(LNET_LOCK_EX);
/* Add peer_ni to global peer table hash, if necessary. */
if (list_empty(&lpni->lpni_hashlist)) {
+ int hash = lnet_nid2peerhash(lpni->lpni_nid);
+
ptable = the_lnet.ln_peer_tables[lpni->lpni_cpt];
- list_add_tail(&lpni->lpni_hashlist,
- &ptable->pt_hash[lnet_nid2peerhash(nid)]);
+ list_add_tail(&lpni->lpni_hashlist, &ptable->pt_hash[hash]);
ptable->pt_version++;
atomic_inc(&ptable->pt_number);
atomic_inc(&lpni->lpni_refcount);
/* Detach the peer_ni from an existing peer, if necessary. */
if (lpni->lpni_peer_net && lpni->lpni_peer_net->lpn_peer != lp)
- lnet_try_destroy_peer_hierarchy_locked(lpni);
+ lnet_peer_detach_peer_ni(lpni);
/* Add peer_ni to peer_net */
lpni->lpni_peer_net = lpn;
/* Add peer to global peer list */
if (list_empty(&lp->lp_on_lnet_peer_list))
list_add_tail(&lp->lp_on_lnet_peer_list, &the_lnet.ln_peers);
+
+ /* Update peer state */
+ spin_lock(&lp->lp_lock);
+ if (flags & LNET_PEER_CONFIGURED) {
+ if (!(lp->lp_state & LNET_PEER_CONFIGURED))
+ lp->lp_state |= LNET_PEER_CONFIGURED;
+ }
+ if (flags & LNET_PEER_MULTI_RAIL) {
+ if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
+ lp->lp_state |= LNET_PEER_MULTI_RAIL;
+ lnet_peer_clr_non_mr_pref_nids(lp);
+ }
+ }
+ spin_unlock(&lp->lp_lock);
+
lnet_net_unlock(LNET_LOCK_EX);
- return 0;
+ CDEBUG(D_NET, "peer %s NID %s flags %#x\n",
+ libcfs_nid2str(lp->lp_primary_nid),
+ libcfs_nid2str(lpni->lpni_nid), flags);
-out_maybe_free_lpn:
- if (list_empty(&lpn->lpn_on_peer_list))
- LIBCFS_FREE(lpn, sizeof(*lpn));
-out_maybe_free_lp:
- if (list_empty(&lp->lp_on_lnet_peer_list))
- LIBCFS_FREE(lp, sizeof(*lp));
-out_enomem:
- return -ENOMEM;
+ return 0;
}
/*
* Create a new peer, with nid as its primary nid.
*
- * It is not an error if the peer already exists, provided that the
- * given nid is the primary NID.
- *
* Call with the lnet_api_mutex held.
*/
static int
-lnet_peer_add(lnet_nid_t nid, bool mr)
+lnet_peer_add(lnet_nid_t nid, unsigned flags)
{
struct lnet_peer *lp;
+ struct lnet_peer_net *lpn;
struct lnet_peer_ni *lpni;
- int rc;
+ int rc = 0;
LASSERT(nid != LNET_NID_ANY);
* lnet_api_mutex is held.
*/
lpni = lnet_find_peer_ni_locked(nid);
- if (!lpni) {
- rc = lnet_peer_setup_hierarchy(NULL, NULL, nid);
- if (rc != 0)
- return rc;
- lpni = lnet_find_peer_ni_locked(nid);
- LASSERT(lpni);
+ if (lpni) {
+ /* A peer with this NID already exists. */
+ lp = lpni->lpni_peer_net->lpn_peer;
+ lnet_peer_ni_decref_locked(lpni);
+ /*
+ * This is an error if the peer was configured and the
+ * primary NID differs or an attempt is made to change
+ * the Multi-Rail flag. Otherwise the assumption is
+ * that an existing peer is being modified.
+ */
+ if (lp->lp_state & LNET_PEER_CONFIGURED) {
+ if (lp->lp_primary_nid != nid)
+ rc = -EEXIST;
+ else if ((lp->lp_state ^ flags) & LNET_PEER_MULTI_RAIL)
+ rc = -EPERM;
+ goto out;
+ }
+ /* Delete and recreate as a configured peer. */
+ lnet_peer_del(lp);
}
- lp = lpni->lpni_peer_net->lpn_peer;
- lnet_peer_ni_decref_locked(lpni);
- /* A found peer must have this primary NID */
- if (lp->lp_primary_nid != nid)
- return -EEXIST;
+ /* Create peer, peer_net, and peer_ni. */
+ rc = -ENOMEM;
+ lp = lnet_peer_alloc(nid);
+ if (!lp)
+ goto out;
+ lpn = lnet_peer_net_alloc(LNET_NIDNET(nid));
+ if (!lpn)
+ goto out_free_lp;
+ lpni = lnet_peer_ni_alloc(nid);
+ if (!lpni)
+ goto out_free_lpn;
- /*
- * If we found an lpni that is not a multi-rail, which could occur
- * if lpni is already created as a non-mr lpni or we just created
- * it, then make sure you indicate that this lpni is a primary mr
- * capable peer.
- *
- * TODO: update flags if necessary
- */
- spin_lock(&lp->lp_lock);
- if (mr && !(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
- lp->lp_state |= LNET_PEER_MULTI_RAIL;
- lnet_peer_clr_non_mr_pref_nids(lp);
- } else if (!mr && (lp->lp_state & LNET_PEER_MULTI_RAIL)) {
- /* The mr state is sticky. */
- CDEBUG(D_NET, "Cannot clear multi-rail flag from peer %s\n",
- libcfs_nid2str(nid));
- }
- spin_unlock(&lp->lp_lock);
+ return lnet_peer_attach_peer_ni(lp, lpn, lpni, flags);
- return 0;
+out_free_lpn:
+ LIBCFS_FREE(lpn, sizeof(*lpn));
+out_free_lp:
+ LIBCFS_FREE(lp, sizeof(*lp));
+out:
+ CDEBUG(D_NET, "peer %s NID flags %#x: %d\n",
+ libcfs_nid2str(nid), flags, rc);
+ return rc;
}
+/*
+ * Add a NID to a peer. Call with ln_api_mutex held.
+ *
+ * Error codes:
+ * -EPERM: Non-DLC addition to a DLC-configured peer.
+ * -EEXIST: The NID was configured by DLC for a different peer.
+ * -ENOMEM: Out of memory.
+ * -ENOTUNIQ: Adding a second peer NID on a single network on a
+ * non-multi-rail peer.
+ */
static int
-lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid, bool mr)
+lnet_peer_add_nid(struct lnet_peer *lp, lnet_nid_t nid, unsigned flags)
{
+ struct lnet_peer_net *lpn;
struct lnet_peer_ni *lpni;
+ int rc = 0;
LASSERT(lp);
LASSERT(nid != LNET_NID_ANY);
- spin_lock(&lp->lp_lock);
- if (!mr && !(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
- spin_unlock(&lp->lp_lock);
- CERROR("Cannot add nid %s to non-multi-rail peer %s\n",
- libcfs_nid2str(nid),
- libcfs_nid2str(lp->lp_primary_nid));
- return -EPERM;
+ /* A configured peer can only be updated through configuration. */
+ if (!(flags & LNET_PEER_CONFIGURED)) {
+ if (lp->lp_state & LNET_PEER_CONFIGURED) {
+ rc = -EPERM;
+ goto out;
+ }
}
- if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
- lp->lp_state |= LNET_PEER_MULTI_RAIL;
- lnet_peer_clr_non_mr_pref_nids(lp);
+ /*
+ * The MULTI_RAIL flag can be set but not cleared, because
+ * that would leave the peer struct in an invalid state.
+ */
+ if (flags & LNET_PEER_MULTI_RAIL) {
+ spin_lock(&lp->lp_lock);
+ if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
+ lp->lp_state |= LNET_PEER_MULTI_RAIL;
+ lnet_peer_clr_non_mr_pref_nids(lp);
+ }
+ spin_unlock(&lp->lp_lock);
+ } else if (lp->lp_state & LNET_PEER_MULTI_RAIL) {
+ rc = -EPERM;
+ goto out;
}
- spin_unlock(&lp->lp_lock);
lpni = lnet_find_peer_ni_locked(nid);
- if (!lpni)
- return lnet_peer_setup_hierarchy(lp, NULL, nid);
+ if (lpni) {
+ /*
+ * A peer_ni already exists. This is only a problem if
+ * it is not connected to this peer and was configured
+ * by DLC.
+ */
+ lnet_peer_ni_decref_locked(lpni);
+ if (lpni->lpni_peer_net->lpn_peer == lp)
+ goto out;
+ if (lnet_peer_ni_is_configured(lpni)) {
+ rc = -EEXIST;
+ goto out;
+ }
+ /* If this is the primary NID, destroy the peer. */
+ if (lnet_peer_ni_is_primary(lpni)) {
+ lnet_peer_del(lpni->lpni_peer_net->lpn_peer);
+ lpni = lnet_peer_ni_alloc(nid);
+ if (!lpni) {
+ rc = -ENOMEM;
+ goto out;
+ }
+ }
+ } else {
+ lpni = lnet_peer_ni_alloc(nid);
+ if (!lpni) {
+ rc = -ENOMEM;
+ goto out;
+ }
+ }
- if (lpni->lpni_peer_net->lpn_peer != lp) {
- struct lnet_peer *lp2 = lpni->lpni_peer_net->lpn_peer;
- CERROR("Cannot add NID %s owned by peer %s to peer %s\n",
- libcfs_nid2str(lpni->lpni_nid),
- libcfs_nid2str(lp2->lp_primary_nid),
- libcfs_nid2str(lp->lp_primary_nid));
- return -EEXIST;
+ /*
+ * Get the peer_net. Check that we're not adding a second
+ * peer_ni on a peer_net of a non-multi-rail peer.
+ */
+ lpn = lnet_peer_get_net_locked(lp, LNET_NIDNET(nid));
+ if (!lpn) {
+ lpn = lnet_peer_net_alloc(LNET_NIDNET(nid));
+ if (!lpn) {
+ rc = -ENOMEM;
+ goto out_free_lpni;
+ }
+ } else if (!(lp->lp_state & LNET_PEER_MULTI_RAIL)) {
+ rc = -ENOTUNIQ;
+ goto out_free_lpni;
}
- CDEBUG(D_NET, "NID %s is already owned by peer %s\n",
- libcfs_nid2str(lpni->lpni_nid),
- libcfs_nid2str(lp->lp_primary_nid));
- return 0;
+ return lnet_peer_attach_peer_ni(lp, lpn, lpni, flags);
+
+out_free_lpni:
+ /* If the peer_ni was allocated above its peer_net pointer is NULL */
+ if (!lpni->lpni_peer_net)
+ LIBCFS_FREE(lpni, sizeof(*lpni));
+out:
+ CDEBUG(D_NET, "peer %s NID %s flags %#x: %d\n",
+ libcfs_nid2str(lp->lp_primary_nid), libcfs_nid2str(nid),
+ flags, rc);
+ return rc;
}
/*
static int
lnet_peer_ni_traffic_add(lnet_nid_t nid, lnet_nid_t pref)
{
+ struct lnet_peer *lp;
+ struct lnet_peer_net *lpn;
struct lnet_peer_ni *lpni;
- int rc;
+ unsigned flags = 0;
+ int rc = 0;
- if (nid == LNET_NID_ANY)
- return -EINVAL;
+ if (nid == LNET_NID_ANY) {
+ rc = -EINVAL;
+ goto out;
+ }
/* lnet_net_lock is not needed here because ln_api_lock is held */
lpni = lnet_find_peer_ni_locked(nid);
- if (!lpni) {
- rc = lnet_peer_setup_hierarchy(NULL, NULL, nid);
- if (rc)
- return rc;
- lpni = lnet_find_peer_ni_locked(nid);
+ if (lpni) {
+ /*
+ * We must have raced with another thread. Since we
+ * know next to nothing about a peer_ni created by
+ * traffic, we just assume everything is ok and
+ * return.
+ */
+ lnet_peer_ni_decref_locked(lpni);
+ goto out;
}
+
+ /* Create peer, peer_net, and peer_ni. */
+ rc = -ENOMEM;
+ lp = lnet_peer_alloc(nid);
+ if (!lp)
+ goto out;
+ lpn = lnet_peer_net_alloc(LNET_NIDNET(nid));
+ if (!lpn)
+ goto out_free_lp;
+ lpni = lnet_peer_ni_alloc(nid);
+ if (!lpni)
+ goto out_free_lpn;
if (pref != LNET_NID_ANY)
lnet_peer_ni_set_non_mr_pref_nid(lpni, pref);
- lnet_peer_ni_decref_locked(lpni);
- return 0;
+ return lnet_peer_attach_peer_ni(lp, lpn, lpni, flags);
+
+out_free_lpn:
+ LIBCFS_FREE(lpn, sizeof(*lpn));
+out_free_lp:
+ LIBCFS_FREE(lp, sizeof(*lp));
+out:
+ CDEBUG(D_NET, "peer %s: %d\n", libcfs_nid2str(nid), rc);
+ return rc;
}
/*
{
struct lnet_peer *lp = NULL;
struct lnet_peer_ni *lpni;
+ unsigned flags;
/* The prim_nid must always be specified */
if (prim_nid == LNET_NID_ANY)
return -EINVAL;
+ flags = LNET_PEER_CONFIGURED;
+ if (mr)
+ flags |= LNET_PEER_MULTI_RAIL;
+
/*
* If nid isn't specified, we must create a new peer with
* prim_nid as its primary nid.
*/
if (nid == LNET_NID_ANY)
- return lnet_peer_add(prim_nid, mr);
+ return lnet_peer_add(prim_nid, flags);
/* Look up the prim_nid, which must exist. */
lpni = lnet_find_peer_ni_locked(prim_nid);
lnet_peer_ni_decref_locked(lpni);
lp = lpni->lpni_peer_net->lpn_peer;
+ /* Peer must have been configured. */
+ if (!(lp->lp_state & LNET_PEER_CONFIGURED)) {
+ CDEBUG(D_NET, "peer %s was not configured\n",
+ libcfs_nid2str(prim_nid));
+ return -ENOENT;
+ }
+
+ /* Primary NID must match */
if (lp->lp_primary_nid != prim_nid) {
CDEBUG(D_NET, "prim_nid %s is not primary for peer %s\n",
libcfs_nid2str(prim_nid),
return -ENODEV;
}
- return lnet_peer_add_nid(lp, nid, mr);
+ /* Multi-Rail flag must match. */
+ if ((lp->lp_state ^ flags) & LNET_PEER_MULTI_RAIL) {
+ CDEBUG(D_NET, "multi-rail state mismatch for peer %s\n",
+ libcfs_nid2str(prim_nid));
+ return -EPERM;
+ }
+
+ return lnet_peer_add_nid(lp, nid, flags);
}
/*
{
struct lnet_peer *lp;
struct lnet_peer_ni *lpni;
+ unsigned flags;
if (prim_nid == LNET_NID_ANY)
return -EINVAL;
if (nid == LNET_NID_ANY || nid == lp->lp_primary_nid)
return lnet_peer_del(lp);
- return lnet_peer_del_nid(lp, nid);
+ flags = LNET_PEER_CONFIGURED;
+ if (lp->lp_state & LNET_PEER_MULTI_RAIL)
+ flags |= LNET_PEER_MULTI_RAIL;
+
+ return lnet_peer_del_nid(lp, nid, flags);
}
void
git://git.whamcloud.com / fs / lustre-release.git / blobdiff