LU-9120 lnet: health error simulation

[fs/lustre-release.git] / lnet / klnds / socklnd / socklnd_cb.c

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index fb16e04..aa32b2a 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -223,7 +223,7 @@ ksocknal_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
                        /* allocated send buffer bytes < computed; infer
                         * something got ACKed */
                        conn->ksnc_tx_deadline = ktime_get_seconds() +
-                                                *ksocknal_tunables.ksnd_timeout;
+                                                lnet_get_lnd_timeout();
                        conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds();
                        conn->ksnc_tx_bufnob = bufnob;
                        smp_mb();
@@ -272,7 +272,7 @@ ksocknal_recv_iov(struct ksock_conn *conn)
 
        conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds();
        conn->ksnc_rx_deadline = ktime_get_seconds() +
-                                *ksocknal_tunables.ksnd_timeout;
+                                lnet_get_lnd_timeout();
        smp_mb();                       /* order with setting rx_started */
        conn->ksnc_rx_started = 1;
 
@@ -316,7 +316,7 @@ ksocknal_recv_kiov(struct ksock_conn *conn)
 
        conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds();
        conn->ksnc_rx_deadline = ktime_get_seconds() +
-                                *ksocknal_tunables.ksnd_timeout;
+                                lnet_get_lnd_timeout();
        smp_mb();                       /* order with setting rx_started */
        conn->ksnc_rx_started = 1;
 
@@ -400,7 +400,8 @@ ksocknal_tx_done(struct lnet_ni *ni, struct ksock_tx *tx, int rc)
 
        if (!rc && (tx->tx_resid != 0 || tx->tx_zc_aborted)) {
                rc = -EIO;
-               hstatus = LNET_MSG_STATUS_LOCAL_ERROR;
+               if (hstatus == LNET_MSG_STATUS_OK)
+                       hstatus = LNET_MSG_STATUS_LOCAL_ERROR;
        }
 
        if (tx->tx_conn != NULL)
@@ -489,7 +490,7 @@ ksocknal_check_zc_req(struct ksock_tx *tx)
 
         /* ZC_REQ is going to be pinned to the peer_ni */
        tx->tx_deadline = ktime_get_seconds() +
-                         *ksocknal_tunables.ksnd_timeout;
+                         lnet_get_lnd_timeout();
 
         LASSERT (tx->tx_msg.ksm_zc_cookies[0] == 0);
 
@@ -533,6 +534,13 @@ static int
 ksocknal_process_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
 {
        int rc;
+       bool error_sim = false;
+
+       if (lnet_send_error_simulation(tx->tx_lnetmsg, &tx->tx_hstatus)) {
+               error_sim = true;
+               rc = -EINVAL;
+               goto simulate_error;
+       }
 
         if (tx->tx_zc_capable && !tx->tx_zc_checked)
                 ksocknal_check_zc_req(tx);
@@ -580,17 +588,21 @@ ksocknal_process_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
                return (rc);
        }
 
+simulate_error:
+
        /* Actual error */
        LASSERT(rc < 0);
 
-       /*
-        * set the health status of the message which determines
-        * whether we should retry the transmit
-        */
-       if (rc == -ETIMEDOUT)
-               tx->tx_hstatus = LNET_MSG_STATUS_REMOTE_TIMEOUT;
-       else
-               tx->tx_hstatus = LNET_MSG_STATUS_LOCAL_ERROR;
+       if (!error_sim) {
+               /*
+               * set the health status of the message which determines
+               * whether we should retry the transmit
+               */
+               if (rc == -ETIMEDOUT)
+                       tx->tx_hstatus = LNET_MSG_STATUS_REMOTE_TIMEOUT;
+               else
+                       tx->tx_hstatus = LNET_MSG_STATUS_LOCAL_ERROR;
+       }
 
        if (!conn->ksnc_closing) {
                switch (rc) {
@@ -770,7 +782,7 @@ ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn)
        if (list_empty(&conn->ksnc_tx_queue) && bufnob == 0) {
                /* First packet starts the timeout */
                conn->ksnc_tx_deadline = ktime_get_seconds() +
-                                        *ksocknal_tunables.ksnd_timeout;
+                                        lnet_get_lnd_timeout();
                if (conn->ksnc_tx_bufnob > 0) /* something got ACKed */
                        conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds();
                conn->ksnc_tx_bufnob = 0;
@@ -947,7 +959,7 @@ ksocknal_launch_packet(struct lnet_ni *ni, struct ksock_tx *tx,
             ksocknal_find_connecting_route_locked (peer_ni) != NULL) {
                 /* the message is going to be pinned to the peer_ni */
                tx->tx_deadline = ktime_get_seconds() +
-                                 *ksocknal_tunables.ksnd_timeout;
+                                 lnet_get_lnd_timeout();
 
                 /* Queue the message until a connection is established */
                list_add_tail(&tx->tx_list, &peer_ni->ksnp_tx_queue);
@@ -1762,7 +1774,7 @@ ksocknal_recv_hello(struct lnet_ni *ni, struct ksock_conn *conn,
        /* socket type set on active connections - not set on passive */
        LASSERT(!active == !(conn->ksnc_type != SOCKLND_CONN_NONE));
 
-       timeout = active ? *ksocknal_tunables.ksnd_timeout :
+       timeout = active ? lnet_get_lnd_timeout() :
                            lnet_acceptor_timeout();
 
        rc = lnet_sock_read(sock, &hello->kshm_magic,
@@ -1897,7 +1909,7 @@ ksocknal_connect(struct ksock_route *route)
         int               retry_later = 0;
         int               rc = 0;
 
-       deadline = ktime_get_seconds() + *ksocknal_tunables.ksnd_timeout;
+       deadline = ktime_get_seconds() + lnet_get_lnd_timeout();
 
        write_lock_bh(&ksocknal_data.ksnd_global_lock);
 
@@ -2636,6 +2648,7 @@ int ksocknal_reaper(void *arg)
                         const int n = 4;
                         const int p = 1;
                         int       chunk = ksocknal_data.ksnd_peer_hash_size;
+                       unsigned int lnd_timeout;
 
                         /* Time to check for timeouts on a few more peers: I do
                          * checks every 'p' seconds on a proportion of the peer_ni
@@ -2644,11 +2657,11 @@ int ksocknal_reaper(void *arg)
                          * timeout on any connection within (n+1)/n times the
                          * timeout interval. */
 
-                        if (*ksocknal_tunables.ksnd_timeout > n * p)
-                                chunk = (chunk * n * p) /
-                                        *ksocknal_tunables.ksnd_timeout;
-                        if (chunk == 0)
-                                chunk = 1;
+                       lnd_timeout = lnet_get_lnd_timeout();
+                       if (lnd_timeout > n * p)
+                               chunk = (chunk * n * p) / lnd_timeout;
+                       if (chunk == 0)
+                               chunk = 1;
 
                         for (i = 0; i < chunk; i++) {
                                 ksocknal_check_peer_timeouts (peer_index);