Branch HEAD

 b=18674
 i=johann
 i=andrew.perepechko
 i=nathan.ruthman
 i=alexey.lyashkov
----------------------------------------------------------------------
 Modified Files:
 	lustre/ChangeLog lustre/include/lustre_import.h
 	lustre/mdt/mdt_handler.c lustre/ptlrpc/import.c
 	lustre/tests/conf-sanity.sh
----------------------------------------------------------------------
Description: client could not reconnect to OST because of an active request.
Details    : The client now retries to reconnect to the same server, if a
	     connect request failed with EBUSY or -EAGAIN. Test case 35b
	     conf-sanity.sh has been created.

diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index 6c8d98b..29d3184 100644 (file)
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -14,6 +14,13 @@ tbd  Sun Microsystems, Inc.
        * File join has been disabled in this release, refer to Bugzilla 16929.
 
 Severity   : normal
        * File join has been disabled in this release, refer to Bugzilla 16929.
 
 Severity   : normal

+Bugzilla   : 18674
+Description: client could not reconnect to OST because of an active request.
+Details    : The client now retries to reconnect to the same server, if a
+            connect request failed with EBUSY or -EAGAIN. Test case 35b
+            conf-sanity.sh has been created.
+
+Severity   : normal

 Bugzilla   : 19557
 Description: Don't allow make backward step on assignin osc next id.
 Details    : race between allocation next id and ll_sync thread can be cause
 Bugzilla   : 19557
 Description: Don't allow make backward step on assignin osc next id.
 Details    : race between allocation next id and ll_sync thread can be cause

diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h
index be05058..7cc9718 100644 (file)
--- a/lustre/include/lustre_import.h
+++ b/lustre/include/lustre_import.h
@@ -181,7 +181,8 @@ struct obd_import {
                                   imp_pingable:1,         /* pingable */
                                   imp_resend_replay:1,    /* resend for replay */
                                   imp_recon_bk:1,         /* turn off reconnect if all failovers fail */
                                   imp_pingable:1,         /* pingable */
                                   imp_resend_replay:1,    /* resend for replay */
                                   imp_recon_bk:1,         /* turn off reconnect if all failovers fail */

-                                  imp_last_recon:1;       /* internally used by above */
+                                  imp_last_recon:1,       /* internally used by above */
+                                  imp_force_reconnect:1;  /* import must be reconnected instead of chouse new connection */

         __u32                     imp_connect_op;
         struct obd_connect_data   imp_connect_data;
         __u64                     imp_connect_flags_orig;
         __u32                     imp_connect_op;
         struct obd_connect_data   imp_connect_data;
         __u64                     imp_connect_flags_orig;

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 0937f72..c6dfae9 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -957,6 +957,7 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
 relock:
                 ma = &info->mti_attr;
 
 relock:
                 ma = &info->mti_attr;
 

+                OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RESEND, obd_timeout*2);

                 mdt_lock_handle_init(lhc);
                 mdt_lock_reg_init(lhc, LCK_PR);
 
                 mdt_lock_handle_init(lhc);
                 mdt_lock_reg_init(lhc, LCK_PR);
 

diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index be0264d..fad264b 100644 (file)
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -499,7 +499,7 @@ static int import_select_connection(struct obd_import *imp)
         }
 
         /* if not found, simply choose the current one */
         }
 
         /* if not found, simply choose the current one */

-        if (!imp_conn) {
+        if (!imp_conn || imp->imp_force_reconnect) {

                 LASSERT(imp->imp_conn_current);
                 imp_conn = imp->imp_conn_current;
                 tried_all = 0;
                 LASSERT(imp->imp_conn_current);
                 imp_conn = imp->imp_conn_current;
                 tried_all = 0;


@@ -779,6 +779,12 @@ static void ptlrpc_maybe_ping_import_soon(struct obd_import *imp)
         EXIT;
 }
 
         EXIT;
 }
 

+static int ptlrpc_busy_reconnect(int rc)
+{
+        return (rc == -EBUSY) || (rc == -EAGAIN);
+}
+
+

 static int ptlrpc_connect_interpret(const struct lu_env *env,
                                     struct ptlrpc_request *request,
                                     void *data, int rc)
 static int ptlrpc_connect_interpret(const struct lu_env *env,
                                     struct ptlrpc_request *request,
                                     void *data, int rc)


@@ -796,18 +802,23 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
                 spin_unlock(&imp->imp_lock);
                 RETURN(0);
         }
                 spin_unlock(&imp->imp_lock);
                 RETURN(0);
         }

-        spin_unlock(&imp->imp_lock);

 
 

-        if (rc)
+        if (rc) {
+                /* if this reconnect to busy export - not need select new target
+                 * for connecting*/
+                if (ptlrpc_busy_reconnect(rc))
+                        imp->imp_force_reconnect = 1;
+                spin_unlock(&imp->imp_lock);

                 GOTO(out, rc);
                 GOTO(out, rc);

+        }

 
         LASSERT(imp->imp_conn_current);
 
         msg_flags = lustre_msg_get_op_flags(request->rq_repmsg);
 
         /* All imports are pingable */
 
         LASSERT(imp->imp_conn_current);
 
         msg_flags = lustre_msg_get_op_flags(request->rq_repmsg);
 
         /* All imports are pingable */

-        spin_lock(&imp->imp_lock);

         imp->imp_pingable = 1;
         imp->imp_pingable = 1;

+        imp->imp_force_reconnect = 0;

 
         if (aa->pcaa_initial_connect) {
                 if (msg_flags & MSG_CONNECT_REPLAYABLE) {
 
         if (aa->pcaa_initial_connect) {
                 if (msg_flags & MSG_CONNECT_REPLAYABLE) {

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 2287f88..e9c63f0 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -1197,7 +1197,7 @@ test_34c() {
 }
 run_test 34c "force umount with failed ost should be normal"
 
 }
 run_test 34c "force umount with failed ost should be normal"
 

-test_35() { # bug 12459
+test_35a() { # bug 12459

        setup
 
        DBG_SAVE="`lctl get_param -n debug`"
        setup
 
        DBG_SAVE="`lctl get_param -n debug`"


@@ -1211,7 +1211,7 @@ test_35() { # bug 12459
        log "Wait for RECONNECT_INTERVAL seconds (10s)"
        sleep 10
 
        log "Wait for RECONNECT_INTERVAL seconds (10s)"
        sleep 10
 

-       MSG="conf-sanity.sh test_35 `date +%F%kh%Mm%Ss`"
+       MSG="conf-sanity.sh test_35a `date +%F%kh%Mm%Ss`"

        $LCTL clear
        log "$MSG"
        log "Stopping the MDT:"
        $LCTL clear
        log "$MSG"
        log "Stopping the MDT:"


@@ -1242,7 +1242,74 @@ test_35() { # bug 12459
        [ "$NEXTCONN" != "0" ] && log "The client didn't try to reconnect to the last active server (tried ${NEXTCONN} instead)" && return 7
        cleanup
 }
        [ "$NEXTCONN" != "0" ] && log "The client didn't try to reconnect to the last active server (tried ${NEXTCONN} instead)" && return 7
        cleanup
 }

-run_test 35 "Reconnect to the last active server first"
+run_test 35a "Reconnect to the last active server first"
+
+test_35b() { # bug 18674
+       remote_mds || { skip "local MDS" && return 0; }
+       setup
+
+       debugsave
+       $LCTL set_param debug="ha"
+       $LCTL clear
+       MSG="conf-sanity.sh test_35b `date +%F%kh%Mm%Ss`"
+       log "$MSG"
+
+       log "Set up a fake failnode for the MDS"
+       FAKENID="127.0.0.2"
+       local device=$(do_facet mds "$LCTL get_param -n devices" | \
+                       awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
+       do_facet mds "$LCTL conf_param ${device}.failover.node=$FAKENID" || \
+               return 1
+
+       local at_max_saved=0
+       # adaptive timeouts may prevent seeing the issue 
+       if at_is_enabled; then
+               at_max_saved=$(at_max_get mds)
+               at_max_set 0 mds client
+       fi
+
+       mkdir -p $MOUNT/testdir
+       touch $MOUNT/testdir/test
+
+       log "Injecting EBUSY on MDS"
+       # Setting OBD_FAIL_MDS_RESEND=0x136
+       do_facet mds "$LCTL set_param fail_loc=0x80000136" || return 2
+
+       log "Stat on a test file"
+       stat $MOUNT/testdir/test
+
+       log "Stop injecting EBUSY on MDS"
+       do_facet mds "$LCTL set_param fail_loc=0" || return 3
+       rm -f $MOUNT/testdir/test
+
+       log "done"
+       # restore adaptive timeout
+       [ $at_max_saved -ne 0 ] && at_max_set $at_max_saved mds client
+
+       $LCTL dk $TMP/lustre-log-$TESTNAME.log
+
+       # retrieve from the log if the client has ever tried to
+       # contact the fake server after the loss of connection
+       FAILCONN=`awk "BEGIN {ret = 0;}
+                      /import_select_connection.*${FSNAME}-MDT0000-mdc.* using connection/ {
+                               ret = 1;
+                               if (\\\$NF ~ /$FAKENID/) {
+                                       ret = 2;
+                                       exit;
+                               }
+                      }
+                      END {print ret}" $TMP/lustre-log-$TESTNAME.log`
+
+       [ "$FAILCONN" == "0" ] && \
+               log "ERROR: The client reconnection has not been triggered" && \
+               return 4
+       [ "$FAILCONN" == "2" ] && \
+               log "ERROR: The client tried to reconnect to the failover server while the primary was busy" && \
+               return 5
+
+        cleanup
+}
+run_test 35b "Continue reconnection retries, if the active server is busy"

 
 test_36() { # 12743
         local rc
 
 test_36() { # 12743
         local rc