Whamcloud - gitweb
LU-15137 socklnd: decrement connection counters on close
[fs/lustre-release.git] / lnet / klnds / socklnd / socklnd.c
index 4c663ea..c2b0a96 100644 (file)
@@ -422,7 +422,9 @@ ksocknal_incr_conn_count(struct ksock_conn_cb *conn_cb,
        switch (type) {
        case SOCKLND_CONN_CONTROL:
                conn_cb->ksnr_ctrl_conn_count++;
-               /* there's a single control connection per peer */
+               /* there's a single control connection per peer,
+                * two in case of loopback
+                */
                conn_cb->ksnr_connected |= BIT(type);
                break;
        case SOCKLND_CONN_BULK_IN:
@@ -448,6 +450,46 @@ ksocknal_incr_conn_count(struct ksock_conn_cb *conn_cb,
               type, conn_cb->ksnr_connected, conn_cb->ksnr_max_conns);
 }
 
+
+static void
+ksocknal_decr_conn_count(struct ksock_conn_cb *conn_cb,
+                        int type)
+{
+       conn_cb->ksnr_conn_count--;
+
+       /* check if all connections of the given type got created */
+       switch (type) {
+       case SOCKLND_CONN_CONTROL:
+               conn_cb->ksnr_ctrl_conn_count--;
+               /* there's a single control connection per peer,
+                * two in case of loopback
+                */
+               if (conn_cb->ksnr_ctrl_conn_count == 0)
+                       conn_cb->ksnr_connected &= ~BIT(type);
+               break;
+       case SOCKLND_CONN_BULK_IN:
+               conn_cb->ksnr_blki_conn_count--;
+               if (conn_cb->ksnr_blki_conn_count < conn_cb->ksnr_max_conns)
+                       conn_cb->ksnr_connected &= ~BIT(type);
+               break;
+       case SOCKLND_CONN_BULK_OUT:
+               conn_cb->ksnr_blko_conn_count--;
+               if (conn_cb->ksnr_blko_conn_count < conn_cb->ksnr_max_conns)
+                       conn_cb->ksnr_connected &= ~BIT(type);
+               break;
+       case SOCKLND_CONN_ANY:
+               if (conn_cb->ksnr_conn_count < conn_cb->ksnr_max_conns)
+                       conn_cb->ksnr_connected &= ~BIT(type);
+               break;
+       default:
+               LBUG();
+               break;
+       }
+
+       CDEBUG(D_NET, "Del conn type %d, ksnr_connected %x ksnr_max_conns %d\n",
+              type, conn_cb->ksnr_connected, conn_cb->ksnr_max_conns);
+}
+
 static void
 ksocknal_associate_cb_conn_locked(struct ksock_conn_cb *conn_cb,
                                  struct ksock_conn *conn)
@@ -1236,6 +1278,8 @@ ksocknal_close_conn_locked(struct ksock_conn *conn, int error)
        struct ksock_peer_ni *peer_ni = conn->ksnc_peer;
        struct ksock_conn_cb *conn_cb;
        struct ksock_conn *conn2;
+       int conn_count;
+       int duplicate_count = 0;
 
        LASSERT(peer_ni->ksnp_error == 0);
        LASSERT(!conn->ksnc_closing);
@@ -1249,21 +1293,29 @@ ksocknal_close_conn_locked(struct ksock_conn *conn, int error)
                /* dissociate conn from cb... */
                LASSERT(!conn_cb->ksnr_deleted);
 
+               conn_count = ksocknal_get_conn_count_by_type(conn_cb,
+                                                            conn->ksnc_type);
                /* connected bit is set only if all connections
                 * of the given type got created
                 */
-               if (ksocknal_get_conn_count_by_type(conn_cb, conn->ksnc_type) ==
-                   conn_cb->ksnr_max_conns)
+               if (conn_count == conn_cb->ksnr_max_conns)
                        LASSERT((conn_cb->ksnr_connected &
                                BIT(conn->ksnc_type)) != 0);
 
-               list_for_each_entry(conn2, &peer_ni->ksnp_conns, ksnc_list) {
-                       if (conn2->ksnc_conn_cb == conn_cb &&
-                           conn2->ksnc_type == conn->ksnc_type)
-                               goto conn2_found;
+               if (conn_count == 1) {
+                       list_for_each_entry(conn2, &peer_ni->ksnp_conns,
+                                           ksnc_list) {
+                               if (conn2->ksnc_conn_cb == conn_cb &&
+                                   conn2->ksnc_type == conn->ksnc_type)
+                                       duplicate_count += 1;
+                       }
+                       if (duplicate_count > 0)
+                               CERROR("Found %d duplicate conns type %d\n",
+                                      duplicate_count,
+                                      conn->ksnc_type);
                }
-               conn_cb->ksnr_connected &= ~BIT(conn->ksnc_type);
-conn2_found:
+               ksocknal_decr_conn_count(conn_cb, conn->ksnc_type);
+
                conn->ksnc_conn_cb = NULL;
 
                /* drop conn's ref on conn_cb */
@@ -1872,11 +1924,15 @@ static int ksocknal_get_link_status(struct net_device *dev)
 
        LASSERT(dev);
 
-       if (!netif_running(dev))
+       if (!netif_running(dev)) {
                ret = 0;
+               CDEBUG(D_NET, "device not running\n");
+       }
        /* Some devices may not be providing link settings */
-       else if (dev->ethtool_ops->get_link)
+       else if (dev->ethtool_ops->get_link) {
                ret = dev->ethtool_ops->get_link(dev);
+               CDEBUG(D_NET, "get_link returns %u\n", ret);
+       }
 
        return ret;
 }
@@ -1885,11 +1941,16 @@ static int
 ksocknal_handle_link_state_change(struct net_device *dev,
                                  unsigned char operstate)
 {
-       struct lnet_ni *ni;
+       struct lnet_ni *ni = NULL;
        struct ksock_net *net;
        struct ksock_net *cnxt;
        int ifindex;
        unsigned char link_down = !(operstate == IF_OPER_UP);
+       struct in_device *in_dev;
+       bool found_ip = false;
+       struct ksock_interface *ksi = NULL;
+       struct sockaddr_in *sa;
+       DECLARE_CONST_IN_IFADDR(ifa);
 
        ifindex = dev->ifindex;
 
@@ -1898,20 +1959,92 @@ ksocknal_handle_link_state_change(struct net_device *dev,
 
        list_for_each_entry_safe(net, cnxt, &ksocknal_data.ksnd_nets,
                                 ksnn_list) {
-               if (net->ksnn_interface.ksni_index != ifindex)
+
+               ksi = &net->ksnn_interface;
+               sa = (void *)&ksi->ksni_addr;
+               found_ip = false;
+
+               if (ksi->ksni_index != ifindex ||
+                   strcmp(ksi->ksni_name, dev->name))
                        continue;
+
                ni = net->ksnn_ni;
-               if (link_down)
+
+               in_dev = __in_dev_get_rtnl(dev);
+               if (!in_dev) {
+                       CDEBUG(D_NET, "Interface %s has no IPv4 status.\n",
+                              dev->name);
+                       CDEBUG(D_NET, "set link fatal state to 1\n");
+                       atomic_set(&ni->ni_fatal_error_on, 1);
+                       continue;
+               }
+               in_dev_for_each_ifa_rtnl(ifa, in_dev) {
+                       if (sa->sin_addr.s_addr == ifa->ifa_local)
+                               found_ip = true;
+               }
+               endfor_ifa(in_dev);
+
+               if (!found_ip) {
+                       CDEBUG(D_NET, "Interface %s has no matching ip\n",
+                              dev->name);
+                       CDEBUG(D_NET, "set link fatal state to 1\n");
+                       atomic_set(&ni->ni_fatal_error_on, 1);
+                       continue;
+               }
+
+               if (link_down) {
+                       CDEBUG(D_NET, "set link fatal state to 1\n");
                        atomic_set(&ni->ni_fatal_error_on, link_down);
-               else
+               } else {
+                       CDEBUG(D_NET, "set link fatal state to %u\n",
+                              (ksocknal_get_link_status(dev) == 0));
                        atomic_set(&ni->ni_fatal_error_on,
                                   (ksocknal_get_link_status(dev) == 0));
+               }
        }
 out:
        return 0;
 }
 
 
+static int
+ksocknal_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event)
+{
+       struct lnet_ni *ni;
+       struct ksock_net *net;
+       struct ksock_net *cnxt;
+       struct net_device *event_netdev = ifa->ifa_dev->dev;
+       int ifindex;
+       struct ksock_interface *ksi = NULL;
+       struct sockaddr_in *sa;
+
+       if (!ksocknal_data.ksnd_nnets)
+               goto out;
+
+       ifindex = event_netdev->ifindex;
+
+       list_for_each_entry_safe(net, cnxt, &ksocknal_data.ksnd_nets,
+                                ksnn_list) {
+
+               ksi = &net->ksnn_interface;
+               sa = (void *)&ksi->ksni_addr;
+
+               if (ksi->ksni_index != ifindex ||
+                   strcmp(ksi->ksni_name, event_netdev->name))
+                       continue;
+
+               if (sa->sin_addr.s_addr == ifa->ifa_local) {
+                       CDEBUG(D_NET, "set link fatal state to %u\n",
+                              (event == NETDEV_DOWN));
+                       ni = net->ksnn_ni;
+                       atomic_set(&ni->ni_fatal_error_on,
+                                  (event == NETDEV_DOWN));
+               }
+       }
+out:
+       return 0;
+}
+
 /************************************
  * Net device notifier event handler
  ************************************/
@@ -1923,6 +2056,9 @@ static int ksocknal_device_event(struct notifier_block *unused,
 
        operstate = dev->operstate;
 
+       CDEBUG(D_NET, "devevent: status=%ld, iface=%s ifindex %d state %u\n",
+              event, dev->name, dev->ifindex, operstate);
+
        switch (event) {
        case NETDEV_UP:
        case NETDEV_DOWN:
@@ -1934,10 +2070,36 @@ static int ksocknal_device_event(struct notifier_block *unused,
        return NOTIFY_OK;
 }
 
-static struct notifier_block ksocknal_notifier_block = {
+/************************************
+ * Inetaddr notifier event handler
+ ************************************/
+static int ksocknal_inetaddr_event(struct notifier_block *unused,
+                                  unsigned long event, void *ptr)
+{
+       struct in_ifaddr *ifa = ptr;
+
+       CDEBUG(D_NET, "addrevent: status %ld ip addr %pI4, netmask %pI4.\n",
+              event, &ifa->ifa_address, &ifa->ifa_mask);
+
+       switch (event) {
+       case NETDEV_UP:
+       case NETDEV_DOWN:
+       case NETDEV_CHANGE:
+               ksocknal_handle_inetaddr_change(ifa, event);
+               break;
+
+       }
+       return NOTIFY_OK;
+}
+
+static struct notifier_block ksocknal_dev_notifier_block = {
        .notifier_call = ksocknal_device_event,
 };
 
+static struct notifier_block ksocknal_inetaddr_notifier_block = {
+       .notifier_call = ksocknal_inetaddr_event,
+};
+
 static void
 ksocknal_base_shutdown(void)
 {
@@ -1949,8 +2111,10 @@ ksocknal_base_shutdown(void)
               libcfs_kmem_read());
        LASSERT (ksocknal_data.ksnd_nnets == 0);
 
-       if (ksocknal_data.ksnd_init == SOCKNAL_INIT_ALL)
-               unregister_netdevice_notifier(&ksocknal_notifier_block);
+       if (ksocknal_data.ksnd_init == SOCKNAL_INIT_ALL) {
+               unregister_netdevice_notifier(&ksocknal_dev_notifier_block);
+               unregister_inetaddr_notifier(&ksocknal_inetaddr_notifier_block);
+       }
 
        switch (ksocknal_data.ksnd_init) {
        default:
@@ -2116,7 +2280,8 @@ ksocknal_base_startup(void)
                 goto failed;
         }
 
-       register_netdevice_notifier(&ksocknal_notifier_block);
+       register_netdevice_notifier(&ksocknal_dev_notifier_block);
+       register_inetaddr_notifier(&ksocknal_inetaddr_notifier_block);
 
         /* flag everything initialised */
         ksocknal_data.ksnd_init = SOCKNAL_INIT_ALL;