LU-1480 lov: avoid subobj's coh_parent race

* during a file lov object initialization, we need protect the access
  and change of its subobj->coh_parent, since it could be another
  layout change race there, which makes an unreferenced lovsub obj in
  the site object hash table.

* dump lovsub objects in the site if the lovsub device reference > 0
  during its finalization phase.

Signed-off-by: Bobi Jam <bobijam.xu@intel.com>
Change-Id: I5d4b79f25a13258cf04f04459f3f1780246b0144
Reviewed-on: http://review.whamcloud.com/6105
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Lai Siyao <lai.siyao@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/lov/lov_object.c b/lustre/lov/lov_object.c
index c5eeb32..230f9d6 100644 (file)
--- a/lustre/lov/lov_object.c
+++ b/lustre/lov/lov_object.c
@@ -123,14 +123,14 @@ static struct cl_object *lov_sub_find(const struct lu_env *env,
 }
 
 static int lov_init_sub(const struct lu_env *env, struct lov_object *lov,
-                        struct cl_object *stripe,
-                        struct lov_layout_raid0 *r0, int idx)
+                       struct cl_object *stripe, struct lov_layout_raid0 *r0,
+                       int idx)
 {
-        struct cl_object_header *hdr;
-        struct cl_object_header *subhdr;
-        struct cl_object_header *parent;
-        struct lov_oinfo        *oinfo;
-        int result;
+       struct cl_object_header *hdr;
+       struct cl_object_header *subhdr;
+       struct cl_object_header *parent;
+       struct lov_oinfo        *oinfo;
+       int result;
 
        if (OBD_FAIL_CHECK(OBD_FAIL_LOV_INIT)) {
                /* For sanity:test_206.
@@ -143,9 +143,8 @@ static int lov_init_sub(const struct lu_env *env, struct lov_object *lov,
                return -EIO;
        }
 
-        hdr    = cl_object_header(lov2cl(lov));
-        subhdr = cl_object_header(stripe);
-        parent = subhdr->coh_parent;
+       hdr    = cl_object_header(lov2cl(lov));
+       subhdr = cl_object_header(stripe);
 
        oinfo = lov->lo_lsm->lsm_oinfo[idx];
        CDEBUG(D_INODE, DFID"@%p[%d] -> "DFID"@%p: ostid: "DOSTID
@@ -154,19 +153,24 @@ static int lov_init_sub(const struct lu_env *env, struct lov_object *lov,
               PFID(&hdr->coh_lu.loh_fid), hdr, POSTID(&oinfo->loi_oi),
               oinfo->loi_ost_idx, oinfo->loi_ost_gen);
 
-        if (parent == NULL) {
-                subhdr->coh_parent = hdr;
-                subhdr->coh_nesting = hdr->coh_nesting + 1;
-                lu_object_ref_add(&stripe->co_lu, "lov-parent", lov);
-                r0->lo_sub[idx] = cl2lovsub(stripe);
-                r0->lo_sub[idx]->lso_super = lov;
-                r0->lo_sub[idx]->lso_index = idx;
-                result = 0;
-        } else {
+       /* reuse ->coh_attr_guard to protect coh_parent change */
+       spin_lock(&subhdr->coh_attr_guard);
+       parent = subhdr->coh_parent;
+       if (parent == NULL) {
+               subhdr->coh_parent = hdr;
+               spin_unlock(&subhdr->coh_attr_guard);
+               subhdr->coh_nesting = hdr->coh_nesting + 1;
+               lu_object_ref_add(&stripe->co_lu, "lov-parent", lov);
+               r0->lo_sub[idx] = cl2lovsub(stripe);
+               r0->lo_sub[idx]->lso_super = lov;
+               r0->lo_sub[idx]->lso_index = idx;
+               result = 0;
+       } else {
                struct lu_object  *old_obj;
                struct lov_object *old_lov;
                unsigned int mask = D_INODE;
 
+               spin_unlock(&subhdr->coh_attr_guard);
                old_obj = lu_object_locate(&parent->coh_lu, &lov_device_type);
                LASSERT(old_obj != NULL);
                old_lov = cl2lov(lu2cl(old_obj));
@@ -185,8 +189,8 @@ static int lov_init_sub(const struct lu_env *env, struct lov_object *lov,
                LU_OBJECT_DEBUG(mask, env, old_obj, "owned.\n");
                LU_OBJECT_HEADER(mask, env, lov2lu(lov), "try to own.\n");
                cl_object_put(env, stripe);
-        }
-        return result;
+       }
+       return result;
 }
 
 static int lov_init_raid0(const struct lu_env *env,

diff --git a/lustre/lov/lovsub_dev.c b/lustre/lov/lovsub_dev.c
index 8f58f4c..0dedfc4 100644 (file)
--- a/lustre/lov/lovsub_dev.c
+++ b/lustre/lov/lovsub_dev.c
@@ -134,12 +134,16 @@ static struct lu_device *lovsub_device_fini(const struct lu_env *env,
 static struct lu_device *lovsub_device_free(const struct lu_env *env,
                                             struct lu_device *d)
 {
-        struct lovsub_device *lsd  = lu2lovsub_dev(d);
-        struct lu_device     *next = cl2lu_dev(lsd->acid_next);
-
-        cl_device_fini(lu2cl_dev(d));
-        OBD_FREE_PTR(lsd);
-        return next;
+       struct lovsub_device *lsd  = lu2lovsub_dev(d);
+       struct lu_device     *next = cl2lu_dev(lsd->acid_next);
+
+       if (cfs_atomic_read(&d->ld_ref) && d->ld_site) {
+               LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
+               lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer);
+       }
+       cl_device_fini(lu2cl_dev(d));
+       OBD_FREE_PTR(lsd);
+       return next;
 }
 
 static int lovsub_req_init(const struct lu_env *env, struct cl_device *dev,