From 018e6e44fd52e12c9d4ff78ca1a5345b12577fd2 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Tue, 18 Sep 2012 17:07:46 +0400 Subject: [PATCH 1/1] LU-1303 osp: osp object operations add object operations to OSP Signed-off-by: Mikhail Pershin Change-Id: I9e460efe2cbfed984cb57e5e77992127b9084556 Reviewed-on: http://review.whamcloud.com/4030 Reviewed-by: Johann Lombardi Tested-by: Hudson Tested-by: Maloo Reviewed-by: Alex Zhuravlev --- lustre/include/lu_object.h | 3 + lustre/obdclass/lu_object.c | 31 +++++ lustre/osp/osp_internal.h | 17 +++ lustre/osp/osp_object.c | 273 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 323 insertions(+), 1 deletion(-) diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index a2d31ae..b33b8b0 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -1337,5 +1337,8 @@ struct lu_kmem_descr { int lu_kmem_init(struct lu_kmem_descr *caches); void lu_kmem_fini(struct lu_kmem_descr *caches); +void lu_object_assign_fid(const struct lu_env *env, struct lu_object *o, + const struct lu_fid *fid); + /** @} lu */ #endif /* __LUSTRE_LU_OBJECT_H */ diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index 40bf6ec..9eef99e 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -2060,3 +2060,34 @@ void lu_kmem_fini(struct lu_kmem_descr *caches) } } EXPORT_SYMBOL(lu_kmem_fini); + +/** + * Temporary solution to be able to assign fid in ->do_create() + * till we have fully-functional OST fids + */ +void lu_object_assign_fid(const struct lu_env *env, struct lu_object *o, + const struct lu_fid *fid) +{ + struct lu_site *s = o->lo_dev->ld_site; + struct lu_fid *old = &o->lo_header->loh_fid; + struct lu_site_bkt_data *bkt; + struct lu_object *shadow; + cfs_waitlink_t waiter; + cfs_hash_t *hs; + cfs_hash_bd_t bd; + __u64 version = 0; + + LASSERT(fid_is_zero(old)); + + hs = s->ls_obj_hash; + cfs_hash_bd_get_and_lock(hs, (void *)fid, &bd, 1); + shadow = htable_lookup(s, &bd, fid, &waiter, &version); + /* supposed to be unique */ + LASSERT(shadow == NULL); + *old = *fid; + bkt = cfs_hash_bd_extra_get(hs, &bd); + cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash); + bkt->lsb_busy++; + cfs_hash_bd_unlock(hs, &bd, 1); +} +EXPORT_SYMBOL(lu_object_assign_fid); diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 541ed00..f76679a 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -242,4 +242,21 @@ void osp_statfs_need_now(struct osp_device *d); /* lproc_osp.c */ void lprocfs_osp_init_vars(struct lprocfs_static_vars *lvars); +/* osp_sync.c */ +/* functions below will be replaced by full versions with osp_sync.c code */ +static inline +int osp_sync_declare_add(const struct lu_env *env, struct osp_object *o, + llog_op_type type, struct thandle *th) +{ + return 0; +} + +static inline +int osp_sync_add(const struct lu_env *env, struct osp_object *o, + llog_op_type type, struct thandle *th, + const struct lu_attr *attr) +{ + return 0; +} + #endif diff --git a/lustre/osp/osp_object.c b/lustre/osp/osp_object.c index cb2f1b4..89bbf9f 100644 --- a/lustre/osp/osp_object.c +++ b/lustre/osp/osp_object.c @@ -48,7 +48,260 @@ #include "osp_internal.h" +static __u64 osp_object_assign_id(const struct lu_env *env, + struct osp_device *d, struct osp_object *o) +{ + struct osp_thread_info *osi = osp_env_info(env); + const struct lu_fid *f = lu_object_fid(&o->opo_obj.do_lu); + + LASSERT(fid_is_zero(f)); + LASSERT(o->opo_reserved); + o->opo_reserved = 0; + + /* assign fid to anonymous object */ + osi->osi_oi.oi_id = osp_precreate_get_id(d); + osi->osi_oi.oi_seq = FID_SEQ_OST_MDT0; + fid_ostid_unpack(&osi->osi_fid, &osi->osi_oi, d->opd_index); + lu_object_assign_fid(env, &o->opo_obj.do_lu, &osi->osi_fid); + + return osi->osi_oi.oi_id; +} + +static int osp_declare_attr_set(const struct lu_env *env, struct dt_object *dt, + const struct lu_attr *attr, struct thandle *th) +{ + struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev); + struct osp_object *o = dt2osp_obj(dt); + int rc = 0; + + ENTRY; + + /* + * Usually we don't allow server stack to manipulate size + * but there is a special case when striping is created + * late, after stripless file got truncated to non-zero. + * + * In this case we do the following: + * + * 1) grab id in declare - this can lead to leaked OST objects + * but we don't currently have proper mechanism and the only + * options we have are to do truncate RPC holding transaction + * open (very bad) or to grab id in declare at cost of leaked + * OST object in same very rare unfortunate case (just bad) + * notice 1.6-2.0 do assignment outside of running transaction + * all the time, meaning many more chances for leaked objects. + * + * 2) send synchronous truncate RPC with just assigned id + */ + LASSERT(attr != NULL); + if (attr->la_valid & LA_SIZE && attr->la_size > 0) { + LASSERT(!dt_object_exists(dt)); + osp_object_assign_id(env, d, o); + rc = osp_object_truncate(env, dt, attr->la_size); + if (rc) + RETURN(rc); + } + + if (!(attr->la_valid & (LA_UID | LA_GID))) + RETURN(0); + + /* + * track all UID/GID changes via llog + */ + rc = osp_sync_declare_add(env, o, MDS_SETATTR64_REC, th); + + RETURN(rc); +} + +static int osp_attr_set(const struct lu_env *env, struct dt_object *dt, + const struct lu_attr *attr, struct thandle *th, + struct lustre_capa *capa) +{ + struct osp_object *o = dt2osp_obj(dt); + int rc = 0; + + ENTRY; + + /* we're interested in uid/gid changes only */ + if (!(attr->la_valid & (LA_UID | LA_GID))) + RETURN(0); + + /* + * once transaction is committed put proper command on + * the queue going to our OST + */ + rc = osp_sync_add(env, o, MDS_SETATTR64_REC, th, attr); + + /* XXX: send new uid/gid to OST ASAP? */ + + RETURN(rc); +} + +static int osp_declare_object_create(const struct lu_env *env, + struct dt_object *dt, + struct lu_attr *attr, + struct dt_allocation_hint *hint, + struct dt_object_format *dof, + struct thandle *th) +{ + struct osp_thread_info *osi = osp_env_info(env); + struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev); + struct osp_object *o = dt2osp_obj(dt); + const struct lu_fid *fid; + int rc = 0; + + ENTRY; + + LASSERT(d->opd_last_used_file); + fid = lu_object_fid(&dt->do_lu); + + /* + * There can be gaps in precreated ids and record to unlink llog + */ + rc = osp_sync_declare_add(env, o, MDS_UNLINK64_REC, th); + + if (unlikely(!fid_is_zero(fid))) { + /* replay case: caller knows fid */ + osi->osi_off = sizeof(osi->osi_id) * d->opd_index; + rc = dt_declare_record_write(env, d->opd_last_used_file, + sizeof(osi->osi_id), osi->osi_off, + th); + RETURN(rc); + } + + /* + * in declaration we need to reserve object so that we don't block + * awaiting precreation RPC to complete + */ + rc = osp_precreate_reserve(env, d); + /* + * we also need to declare update to local "last used id" file for + * recovery if object isn't used for a reason, we need to release + * reservation, this can be made in osd_object_release() + */ + if (rc == 0) { + /* mark id is reserved: in create we don't want to talk + * to OST */ + LASSERT(o->opo_reserved == 0); + o->opo_reserved = 1; + + /* common for all OSPs file hystorically */ + osi->osi_off = sizeof(osi->osi_id) * d->opd_index; + rc = dt_declare_record_write(env, d->opd_last_used_file, + sizeof(osi->osi_id), osi->osi_off, + th); + } else { + /* not needed in the cache anymore */ + cfs_set_bit(LU_OBJECT_HEARD_BANSHEE, + &dt->do_lu.lo_header->loh_flags); + } + RETURN(rc); +} + +static int osp_object_create(const struct lu_env *env, struct dt_object *dt, + struct lu_attr *attr, + struct dt_allocation_hint *hint, + struct dt_object_format *dof, struct thandle *th) +{ + struct osp_thread_info *osi = osp_env_info(env); + struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev); + struct osp_object *o = dt2osp_obj(dt); + int rc = 0; + + ENTRY; + + if (o->opo_reserved) { + /* regular case, id is assigned holding transaction open */ + osi->osi_id = osp_object_assign_id(env, d, o); + } else { + /* special case, id was assigned outside of transaction + * see comments in osp_declare_attr_set */ + rc = fid_ostid_pack(lu_object_fid(&dt->do_lu), &osi->osi_oi); + LASSERT(rc == 0); + osi->osi_id = ostid_id(&osi->osi_oi); + cfs_spin_lock(&d->opd_pre_lock); + osp_update_last_id(d, osi->osi_id); + cfs_spin_unlock(&d->opd_pre_lock); + } + + LASSERT(osi->osi_id); + + /* + * it's OK if the import is inactive by this moment - id was created + * by OST earlier, we just need to maintain it consistently on the disk + * once import is reconnected, OSP will claim this and other objects + * used and OST either keep them, if they exist or recreate + */ + + /* we might have lost precreated objects */ + if (unlikely(d->opd_gap_count) > 0) { + cfs_spin_lock(&d->opd_pre_lock); + if (d->opd_gap_count > 0) { + int count = d->opd_gap_count; + + osi->osi_oi.oi_id = d->opd_gap_start; + d->opd_gap_count = 0; + cfs_spin_unlock(&d->opd_pre_lock); + + CDEBUG(D_HA, "Found gap "LPU64"+%d in objids\n", + d->opd_gap_start, count); + /* real gap handling is disabled intil ORI-692 will be + * fixed, now we only report gaps */ + } else { + cfs_spin_unlock(&d->opd_pre_lock); + } + } + + osp_objid_buf_prep(osi, d, d->opd_index); + rc = dt_record_write(env, d->opd_last_used_file, &osi->osi_lb, + &osi->osi_off, th); + + RETURN(rc); +} + +static int osp_declare_object_destroy(const struct lu_env *env, + struct dt_object *dt, struct thandle *th) +{ + struct osp_object *o = dt2osp_obj(dt); + int rc = 0; + + ENTRY; + + /* + * track objects to be destroyed via llog + */ + rc = osp_sync_declare_add(env, o, MDS_UNLINK64_REC, th); + + RETURN(rc); +} + +static int osp_object_destroy(const struct lu_env *env, struct dt_object *dt, + struct thandle *th) +{ + struct osp_object *o = dt2osp_obj(dt); + int rc = 0; + + ENTRY; + + /* + * once transaction is committed put proper command on + * the queue going to our OST + */ + rc = osp_sync_add(env, o, MDS_UNLINK64_REC, th, NULL); + + /* not needed in cache any more */ + cfs_set_bit(LU_OBJECT_HEARD_BANSHEE, &dt->do_lu.lo_header->loh_flags); + + RETURN(rc); +} + struct dt_object_operations osp_obj_ops = { + .do_declare_attr_set = osp_declare_attr_set, + .do_attr_set = osp_attr_set, + .do_declare_create = osp_declare_object_create, + .do_create = osp_object_create, + .do_declare_destroy = osp_declare_object_destroy, + .do_destroy = osp_object_destroy, }; static int osp_object_init(const struct lu_env *env, struct lu_object *o, @@ -73,7 +326,25 @@ static void osp_object_free(const struct lu_env *env, struct lu_object *o) static void osp_object_release(const struct lu_env *env, struct lu_object *o) { - return; + struct osp_object *po = lu2osp_obj(o); + struct osp_device *d = lu2osp_dev(o->lo_dev); + + ENTRY; + + /* + * release reservation if object was declared but not created + * this may require lu_object_put() in LOD + */ + if (unlikely(po->opo_reserved)) { + LASSERT(d->opd_pre_reserved > 0); + cfs_spin_lock(&d->opd_pre_lock); + d->opd_pre_reserved--; + cfs_spin_unlock(&d->opd_pre_lock); + + /* not needed in cache any more */ + cfs_set_bit(LU_OBJECT_HEARD_BANSHEE, &o->lo_header->loh_flags); + } + EXIT; } static int osp_object_print(const struct lu_env *env, void *cookie, -- 1.8.3.1