LU-3275 osd-ldiskfs: re-order osd device init/fini

There was race condition between the background OI scrub thread and
osd_device_init0: OI scrub thread may try to access non-initialized
osd_device::od_ost_map. So the osd initization/start process should
NOT trigger OI scrub until all the OI (OI files, /O, and ect.) have
been initialized. Reverse ordre for osd divice fini.

Test-Parameters: testlist=sanity-scrub

Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: I99a2a72ace947a0f79ace5c6f445cf896d884e63
Reviewed-on: http://review.whamcloud.com/6267
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Tested-by: Hudson
Reviewed-by: wangdi <di.wang@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index b34bda8..f5d5718 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -5224,20 +5224,33 @@ static int osd_shutdown(const struct lu_env *env, struct osd_device *o)
 {
        ENTRY;
 
-       osd_scrub_cleanup(env, o);
+       /* shutdown quota slave instance associated with the device */
+       if (o->od_quota_slave != NULL) {
+               qsd_fini(env, o->od_quota_slave);
+               o->od_quota_slave = NULL;
+       }
+
+       RETURN(0);
+}
+
+static void osd_umount(const struct lu_env *env, struct osd_device *o)
+{
+       ENTRY;
 
        if (o->od_fsops) {
                fsfilt_put_ops(o->od_fsops);
                o->od_fsops = NULL;
        }
 
-       /* shutdown quota slave instance associated with the device */
-       if (o->od_quota_slave != NULL) {
-               qsd_fini(env, o->od_quota_slave);
-               o->od_quota_slave = NULL;
+       if (o->od_mnt != NULL) {
+               shrink_dcache_sb(osd_sb(o));
+               osd_sync(env, &o->od_dt_dev);
+
+               mntput(o->od_mnt);
+               o->od_mnt = NULL;
        }
 
-       RETURN(0);
+       EXIT;
 }
 
 static int osd_mount(const struct lu_env *env,
@@ -5339,30 +5352,18 @@ out:
 }
 
 static struct lu_device *osd_device_fini(const struct lu_env *env,
-                                         struct lu_device *d)
+                                        struct lu_device *d)
 {
-        int rc;
-        ENTRY;
-
-       rc = osd_shutdown(env, osd_dev(d));
-
-       osd_obj_map_fini(osd_dev(d));
-
-        shrink_dcache_sb(osd_sb(osd_dev(d)));
-        osd_sync(env, lu2dt_dev(d));
-
-        rc = osd_procfs_fini(osd_dev(d));
-        if (rc) {
-                CERROR("proc fini error %d \n", rc);
-                RETURN (ERR_PTR(rc));
-        }
+       struct osd_device *o = osd_dev(d);
+       ENTRY;
 
-       if (osd_dev(d)->od_mnt) {
-               mntput(osd_dev(d)->od_mnt);
-               osd_dev(d)->od_mnt = NULL;
-       }
+       osd_procfs_fini(o);
+       osd_shutdown(env, o);
+       osd_scrub_cleanup(env, o);
+       osd_obj_map_fini(o);
+       osd_umount(env, o);
 
-        RETURN(NULL);
+       RETURN(NULL);
 }
 
 static int osd_device_init0(const struct lu_env *env,
@@ -5400,12 +5401,6 @@ static int osd_device_init0(const struct lu_env *env,
        if (rc)
                GOTO(out_capa, rc);
 
-       CFS_INIT_LIST_HEAD(&o->od_ios_list);
-       /* setup scrub, including OI files initialization */
-       rc = osd_scrub_setup(env, o);
-       if (rc < 0)
-               GOTO(out_mnt, rc);
-
        cplen = strlcpy(o->od_svname, lustre_cfg_string(cfg, 4),
                        sizeof(o->od_svname));
        if (cplen >= sizeof(o->od_svname)) {
@@ -5415,22 +5410,28 @@ static int osd_device_init0(const struct lu_env *env,
 
        rc = osd_obj_map_init(env, o);
        if (rc != 0)
-               GOTO(out_scrub, rc);
+               GOTO(out_mnt, rc);
 
        rc = lu_site_init(&o->od_site, l);
-       if (rc)
+       if (rc != 0)
                GOTO(out_compat, rc);
        o->od_site.ls_bottom_dev = l;
 
        rc = lu_site_init_finish(&o->od_site);
-       if (rc)
+       if (rc != 0)
+               GOTO(out_site, rc);
+
+       CFS_INIT_LIST_HEAD(&o->od_ios_list);
+       /* setup scrub, including OI files initialization */
+       rc = osd_scrub_setup(env, o);
+       if (rc < 0)
                GOTO(out_site, rc);
 
        rc = osd_procfs_init(o, o->od_svname);
        if (rc != 0) {
                CERROR("%s: can't initialize procfs: rc = %d\n",
                       o->od_svname, rc);
-               GOTO(out_site, rc);
+               GOTO(out_scrub, rc);
        }
 
        LASSERT(l->ld_site->ls_linkage.next && l->ld_site->ls_linkage.prev);
@@ -5445,23 +5446,21 @@ static int osd_device_init0(const struct lu_env *env,
        }
 
        RETURN(0);
+
 out_procfs:
        osd_procfs_fini(o);
+out_scrub:
+       osd_scrub_cleanup(env, o);
 out_site:
        lu_site_fini(&o->od_site);
 out_compat:
        osd_obj_map_fini(o);
-out_scrub:
-       osd_scrub_cleanup(env, o);
 out_mnt:
-       osd_oi_fini(info, o);
-       osd_shutdown(env, o);
-       mntput(o->od_mnt);
-       o->od_mnt = NULL;
+       osd_umount(env, o);
 out_capa:
        cleanup_capa_hash(o->od_capa_hash);
 out:
-       RETURN(rc);
+       return rc;
 }
 
 static struct lu_device *osd_device_alloc(const struct lu_env *env,

diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h
index 9c98789..85c2656 100644 (file)
--- a/lustre/osd-ldiskfs/osd_internal.h
+++ b/lustre/osd-ldiskfs/osd_internal.h
@@ -745,7 +745,8 @@ static inline struct osd_oi *osd_fid2oi(struct osd_device *osd,
 {
        LASSERTF(!fid_is_idif(fid), DFID"\n", PFID(fid));
        LASSERTF(!fid_is_last_id(fid), DFID"\n", PFID(fid));
-       LASSERT(osd->od_oi_table != NULL && osd->od_oi_count >= 1);
+       LASSERTF(osd->od_oi_table != NULL && osd->od_oi_count >= 1,
+                DFID"\n", PFID(fid));
        /* It can work even od_oi_count equals to 1 although it's unexpected,
         * the only reason we set it to 1 is for performance measurement */
        return osd->od_oi_table[osd_oi_fid2idx(osd, fid)];

diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c
index 6e9c37b..71c183f 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.c
+++ b/lustre/osd-ldiskfs/osd_scrub.c
@@ -1177,6 +1177,14 @@ static const struct osd_lf_map osd_lf_maps[] = {
        { QSD_DIR, { 0, 0, 0 }, OLF_SCAN_SUBITEMS,
                osd_ios_general_scan, osd_ios_varfid_fill },
 
+       /* seq-1-lastid */
+       { "seq-1-lastid", { FID_SEQ_LLOG, 1, 0 }, 0,
+               NULL, NULL },
+
+       /* seq-a-lastid */
+       { "seq-a-lastid", { FID_SEQ_LLOG_NAME, 1, 0 }, 0,
+               NULL, NULL },
+
        /* seq-200000003-lastid */
        { "seq-200000003-lastid", { FID_SEQ_LOCAL_NAME, 1, 0 }, 0,
                NULL, NULL },