From 8bec3572c9a360ae44830358fc2441ae5b79baa4 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Wed, 1 May 2013 19:22:47 +0800 Subject: [PATCH] LU-3275 osd-ldiskfs: re-order osd device init/fini There was race condition between the background OI scrub thread and osd_device_init0: OI scrub thread may try to access non-initialized osd_device::od_ost_map. So the osd initization/start process should NOT trigger OI scrub until all the OI (OI files, /O, and ect.) have been initialized. Reverse ordre for osd divice fini. Test-Parameters: testlist=sanity-scrub Signed-off-by: Fan Yong Change-Id: I99a2a72ace947a0f79ace5c6f445cf896d884e63 Reviewed-on: http://review.whamcloud.com/6267 Reviewed-by: Andreas Dilger Tested-by: Hudson Reviewed-by: wangdi Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/osd-ldiskfs/osd_handler.c | 87 +++++++++++++++++++-------------------- lustre/osd-ldiskfs/osd_internal.h | 3 +- lustre/osd-ldiskfs/osd_scrub.c | 8 ++++ 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index b34bda8..f5d5718 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -5224,20 +5224,33 @@ static int osd_shutdown(const struct lu_env *env, struct osd_device *o) { ENTRY; - osd_scrub_cleanup(env, o); + /* shutdown quota slave instance associated with the device */ + if (o->od_quota_slave != NULL) { + qsd_fini(env, o->od_quota_slave); + o->od_quota_slave = NULL; + } + + RETURN(0); +} + +static void osd_umount(const struct lu_env *env, struct osd_device *o) +{ + ENTRY; if (o->od_fsops) { fsfilt_put_ops(o->od_fsops); o->od_fsops = NULL; } - /* shutdown quota slave instance associated with the device */ - if (o->od_quota_slave != NULL) { - qsd_fini(env, o->od_quota_slave); - o->od_quota_slave = NULL; + if (o->od_mnt != NULL) { + shrink_dcache_sb(osd_sb(o)); + osd_sync(env, &o->od_dt_dev); + + mntput(o->od_mnt); + o->od_mnt = NULL; } - RETURN(0); + EXIT; } static int osd_mount(const struct lu_env *env, @@ -5339,30 +5352,18 @@ out: } static struct lu_device *osd_device_fini(const struct lu_env *env, - struct lu_device *d) + struct lu_device *d) { - int rc; - ENTRY; - - rc = osd_shutdown(env, osd_dev(d)); - - osd_obj_map_fini(osd_dev(d)); - - shrink_dcache_sb(osd_sb(osd_dev(d))); - osd_sync(env, lu2dt_dev(d)); - - rc = osd_procfs_fini(osd_dev(d)); - if (rc) { - CERROR("proc fini error %d \n", rc); - RETURN (ERR_PTR(rc)); - } + struct osd_device *o = osd_dev(d); + ENTRY; - if (osd_dev(d)->od_mnt) { - mntput(osd_dev(d)->od_mnt); - osd_dev(d)->od_mnt = NULL; - } + osd_procfs_fini(o); + osd_shutdown(env, o); + osd_scrub_cleanup(env, o); + osd_obj_map_fini(o); + osd_umount(env, o); - RETURN(NULL); + RETURN(NULL); } static int osd_device_init0(const struct lu_env *env, @@ -5400,12 +5401,6 @@ static int osd_device_init0(const struct lu_env *env, if (rc) GOTO(out_capa, rc); - CFS_INIT_LIST_HEAD(&o->od_ios_list); - /* setup scrub, including OI files initialization */ - rc = osd_scrub_setup(env, o); - if (rc < 0) - GOTO(out_mnt, rc); - cplen = strlcpy(o->od_svname, lustre_cfg_string(cfg, 4), sizeof(o->od_svname)); if (cplen >= sizeof(o->od_svname)) { @@ -5415,22 +5410,28 @@ static int osd_device_init0(const struct lu_env *env, rc = osd_obj_map_init(env, o); if (rc != 0) - GOTO(out_scrub, rc); + GOTO(out_mnt, rc); rc = lu_site_init(&o->od_site, l); - if (rc) + if (rc != 0) GOTO(out_compat, rc); o->od_site.ls_bottom_dev = l; rc = lu_site_init_finish(&o->od_site); - if (rc) + if (rc != 0) + GOTO(out_site, rc); + + CFS_INIT_LIST_HEAD(&o->od_ios_list); + /* setup scrub, including OI files initialization */ + rc = osd_scrub_setup(env, o); + if (rc < 0) GOTO(out_site, rc); rc = osd_procfs_init(o, o->od_svname); if (rc != 0) { CERROR("%s: can't initialize procfs: rc = %d\n", o->od_svname, rc); - GOTO(out_site, rc); + GOTO(out_scrub, rc); } LASSERT(l->ld_site->ls_linkage.next && l->ld_site->ls_linkage.prev); @@ -5445,23 +5446,21 @@ static int osd_device_init0(const struct lu_env *env, } RETURN(0); + out_procfs: osd_procfs_fini(o); +out_scrub: + osd_scrub_cleanup(env, o); out_site: lu_site_fini(&o->od_site); out_compat: osd_obj_map_fini(o); -out_scrub: - osd_scrub_cleanup(env, o); out_mnt: - osd_oi_fini(info, o); - osd_shutdown(env, o); - mntput(o->od_mnt); - o->od_mnt = NULL; + osd_umount(env, o); out_capa: cleanup_capa_hash(o->od_capa_hash); out: - RETURN(rc); + return rc; } static struct lu_device *osd_device_alloc(const struct lu_env *env, diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 9c98789..85c2656 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -745,7 +745,8 @@ static inline struct osd_oi *osd_fid2oi(struct osd_device *osd, { LASSERTF(!fid_is_idif(fid), DFID"\n", PFID(fid)); LASSERTF(!fid_is_last_id(fid), DFID"\n", PFID(fid)); - LASSERT(osd->od_oi_table != NULL && osd->od_oi_count >= 1); + LASSERTF(osd->od_oi_table != NULL && osd->od_oi_count >= 1, + DFID"\n", PFID(fid)); /* It can work even od_oi_count equals to 1 although it's unexpected, * the only reason we set it to 1 is for performance measurement */ return osd->od_oi_table[osd_oi_fid2idx(osd, fid)]; diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c index 6e9c37b..71c183f 100644 --- a/lustre/osd-ldiskfs/osd_scrub.c +++ b/lustre/osd-ldiskfs/osd_scrub.c @@ -1177,6 +1177,14 @@ static const struct osd_lf_map osd_lf_maps[] = { { QSD_DIR, { 0, 0, 0 }, OLF_SCAN_SUBITEMS, osd_ios_general_scan, osd_ios_varfid_fill }, + /* seq-1-lastid */ + { "seq-1-lastid", { FID_SEQ_LLOG, 1, 0 }, 0, + NULL, NULL }, + + /* seq-a-lastid */ + { "seq-a-lastid", { FID_SEQ_LLOG_NAME, 1, 0 }, 0, + NULL, NULL }, + /* seq-200000003-lastid */ { "seq-200000003-lastid", { FID_SEQ_LOCAL_NAME, 1, 0 }, 0, NULL, NULL }, -- 1.8.3.1