LU-5299 obdclass: avoid race during Server device start

Handle concurrent starts for same device (multiple mounts, ...).
But allows for separate nosvc and nomgs case.
Also add a specific test of concurent MDT/OST start with an
artificial delay to verify.

Signed-off-by: Bruno Faccini <bruno.faccini@intel.com>
Change-Id: I442819a5b865ed3e98477f9d2602efc4d09d7860
Reviewed-on: http://review.whamcloud.com/11139
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Bobi Jam <bobijam@gmail.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/obdclass/obd_mount_server.c b/lustre/obdclass/obd_mount_server.c
index fba448d..f0b169e 100644 (file)
--- a/lustre/obdclass/obd_mount_server.c
+++ b/lustre/obdclass/obd_mount_server.c
@@ -1668,6 +1668,7 @@ static int osd_start(struct lustre_sb_info *lsi, unsigned long mflags)
        struct dt_device_param    p;
        char                      flagstr[16];
        int                       rc;
+       bool                     already_started = 0;
        ENTRY;
 
        CDEBUG(D_MOUNT,
@@ -1690,15 +1691,27 @@ static int osd_start(struct lustre_sb_info *lsi, unsigned long mflags)
                        GOTO(out, rc);
                obd = class_name2obd(lsi->lsi_osd_obdname);
                LASSERT(obd);
+       } else {
+               CDEBUG(D_MOUNT, "%s already started\n", lsi->lsi_osd_obdname);
+               already_started = 1;
+               /* but continue setup to allow special case of MDT and internal
+                * MGT being started separately, that will be identified in
+                * caller server_fill_super().
+                */
        }
 
        rc = obd_connect(NULL, &lsi->lsi_osd_exp,
                         obd, &obd->obd_uuid, NULL, NULL);
+
+       OBD_FAIL_TIMEOUT(OBD_FAIL_TGT_DELAY_CONNECT, 10);
+
        if (rc) {
-               obd->obd_force = 1;
-               class_manual_cleanup(obd);
-               lsi->lsi_dt_dev = NULL;
-               RETURN(rc);
+               if (!already_started) {
+                       obd->obd_force = 1;
+                       class_manual_cleanup(obd);
+                       lsi->lsi_dt_dev = NULL;
+               }
+               GOTO(out, rc);
        }
 
        LASSERT(obd->obd_lu_dev);
@@ -1712,7 +1725,7 @@ static int osd_start(struct lustre_sb_info *lsi, unsigned long mflags)
 
        dt_conf_get(NULL, lsi->lsi_dt_dev, &p);
 out:
-       RETURN(rc);
+       RETURN(already_started ? -EALREADY : rc);
 }
 
 /** Fill in the superblock info for a Lustre server.
@@ -1732,7 +1745,9 @@ int server_fill_super(struct super_block *sb)
 
        /* Start low level OSD */
        rc = osd_start(lsi, sb->s_flags);
-       if (rc) {
+       /* Handle separate nosvc and nomgs case */
+       if (rc && ((rc != -EALREADY) || !(lsi->lsi_lmd->lmd_flags &
+                                       (LMD_FLG_NOSVC|LMD_FLG_NOMGS)))) {
                CERROR("Unable to start osd on %s: %d\n",
                       lsi->lsi_lmd->lmd_dev, rc);
                lustre_put_lsi(sb);

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 95cc423..c4d2136 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -2564,6 +2564,94 @@ test_41b() {
 }
 run_test 41b "mount mds with --nosvc and --nomgs on first mount"
 
+test_41c() {
+       cleanup
+       # MDT concurent start
+       #define OBD_FAIL_TGT_DELAY_CONNECT 0x703
+       do_facet $SINGLEMDS "lctl set_param fail_loc=0x703"
+       start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS &
+       local pid=$!
+       sleep 2
+       do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
+       start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS &
+       local pid2=$!
+       wait $pid2
+       local rc2=$?
+       wait $pid
+       local rc=$?
+       if [ $rc == 0 ] && [ $rc2 == 114 ]; then
+               echo "1st MDT start succeed"
+               echo "2nd MDT start failed with EALREADY"
+       elif [ $rc2 == 0 ] && [ $rc == 114 ]; then
+               echo "1st MDT start failed with EALREADY"
+               echo "2nd MDT start succeed"
+       else
+               stop mds1 -f
+               error "unexpected concurent MDT mounts result, rc=$rc rc2=$rc2"
+       fi
+
+       # OST concurent start
+       #define OBD_FAIL_TGT_DELAY_CONNECT 0x703
+       do_facet ost1 "lctl set_param fail_loc=0x703"
+       start ost1 $(ostdevname 1) $OST_MOUNT_OPTS &
+       pid=$!
+       sleep 2
+       do_facet ost1 "lctl set_param fail_loc=0x0"
+       start ost1 $(ostdevname 1) $OST_MOUNT_OPTS &
+       pid2=$!
+       wait $pid2
+       rc2=$?
+       wait $pid
+       rc=$?
+       if [ $rc == 0 ] && [ $rc2 == 114 ]; then
+               echo "1st OST start succeed"
+               echo "2nd OST start failed with EALREADY"
+       elif [ $rc2 == 0 ] && [ $rc == 114 ]; then
+               echo "1st OST start failed with EALREADY"
+               echo "2nd OST start succeed"
+       else
+               stop mds1 -f
+               stop ost1 -f
+               error "unexpected concurent OST mounts result, rc=$rc rc2=$rc2"
+       fi
+       # cleanup
+       stop mds1 -f
+       stop ost1 -f
+
+       # verify everything ok
+       start_mds
+       if [ $? != 0 ]
+       then
+               stop mds1 -f
+               error "MDT(s) start failed"
+       fi
+
+       start_ost
+       if [ $? != 0 ]
+       then
+               stop mds1 -f
+               stop ost1 -f
+               error "OST(s) start failed"
+       fi
+
+       mount_client $MOUNT
+       if [ $? != 0 ]
+       then
+               stop mds1 -f
+               stop ost1 -f
+               error "client start failed"
+       fi
+       check_mount
+       if [ $? != 0 ]
+       then
+               stop mds1 -f
+               stop ost1 -f
+               error "client mount failed"
+       fi
+       cleanup
+}
+run_test 41c "concurent mounts of MDT/OST should all fail but one"
+
 test_42() { #bug 14693
        setup
        check_mount || error "client was not mounted"