X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fmdd%2Fmdd_orphans.c;h=442452c1f13e9754e13e452d0ec55f43312dca2b;hb=b69b7de30c3977cb69a741099218bc4a81752717;hp=2ac658b0580fb353772f577ad688544ef0cf8308;hpb=6c8ed1d8edb0f6fe1d79172f7fa4a0fcf5053dfe;p=fs%2Flustre-release.git diff --git a/lustre/mdd/mdd_orphans.c b/lustre/mdd/mdd_orphans.c index 2ac658b..442452c 100644 --- a/lustre/mdd/mdd_orphans.c +++ b/lustre/mdd/mdd_orphans.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -26,8 +24,10 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2011, 2013, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -38,11 +38,9 @@ * Orphan handling code * * Author: Mike Pershin + * Pravin B Shelar */ -#ifndef EXPORT_SYMTAB -# define EXPORT_SYMTAB -#endif #define DEBUG_SUBSYSTEM S_MDS #include @@ -52,156 +50,512 @@ #include #include "mdd_internal.h" -const char orph_index_name[] = "orphans"; - -static const struct dt_index_features orph_index_features = { - .dif_flags = DT_IND_UPDATE, - .dif_keysize_min = sizeof(struct orph_key), - .dif_keysize_max = sizeof(struct orph_key), - .dif_recsize_min = sizeof(loff_t), - .dif_recsize_max = sizeof(loff_t) -}; +const char orph_index_name[] = "PENDING"; +static const char dotdot[] = ".."; enum { ORPH_OP_UNLINK, ORPH_OP_TRUNCATE }; -static struct orph_key *orph_key_fill(const struct lu_env *env, - const struct lu_fid *lf, __u32 op) +#define ORPHAN_FILE_NAME_FORMAT "%016llx:%08x:%08x:%2x" +#define ORPHAN_FILE_NAME_FORMAT_18 "%llx:%08x" + +static struct dt_key* orph_key_fill(const struct lu_env *env, + const struct lu_fid *lf, __u32 op) { - struct orph_key *key = &mdd_env_info(env)->mti_orph_key; + char *key = mdd_env_info(env)->mti_key; + int rc; + LASSERT(key); - fid_cpu_to_be(&key->ok_fid, lf); - key->ok_op = cpu_to_be32(op); - return key; + rc = snprintf(key, NAME_MAX + 1, ORPHAN_FILE_NAME_FORMAT, + (long long unsigned int)fid_seq(lf), + fid_oid(lf), fid_ver(lf), op); + if (rc > 0) + return (struct dt_key*) key; + else + return ERR_PTR(rc); } -static int orph_index_insert(const struct lu_env *env, - struct mdd_object *obj, __u32 op, - loff_t *offset, struct thandle *th) +static struct dt_key* orph_key_fill_18(const struct lu_env *env, + const struct lu_fid *lf) { - struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); - struct dt_object *dor = mdd->mdd_orphans; - struct orph_key *key = orph_key_fill(env, mdo2fid(obj), op); + char *key = mdd_env_info(env)->mti_key; int rc; + + LASSERT(key); + rc = snprintf(key, NAME_MAX + 1, ORPHAN_FILE_NAME_FORMAT_18, + (unsigned long long)fid_seq(lf), fid_oid(lf)); + if (rc > 0) + return (struct dt_key*) key; + else + return ERR_PTR(rc); +} + +static inline void mdd_orphan_write_lock(const struct lu_env *env, + struct mdd_device *mdd) +{ + + struct dt_object *dor = mdd->mdd_orphans; + dor->do_ops->do_write_lock(env, dor, MOR_TGT_ORPHAN); +} + +static inline void mdd_orphan_write_unlock(const struct lu_env *env, + struct mdd_device *mdd) +{ + + struct dt_object *dor = mdd->mdd_orphans; + dor->do_ops->do_write_unlock(env, dor); +} + +static inline int mdd_orphan_insert_obj(const struct lu_env *env, + struct mdd_device *mdd, + struct mdd_object *obj, + __u32 op, struct thandle *th) +{ + struct dt_insert_rec *rec = &mdd_env_info(env)->mti_dt_rec; + struct dt_object *dor = mdd->mdd_orphans; + const struct lu_fid *lf = mdo2fid(obj); + struct dt_key *key = orph_key_fill(env, lf, op); + + rec->rec_fid = lf; + rec->rec_type = mdd_object_type(obj); + + return dor->do_index_ops->dio_insert(env, dor, + (const struct dt_rec *)rec, + key, th, BYPASS_CAPA, 1); +} + +static inline int mdd_orphan_delete_obj(const struct lu_env *env, + struct mdd_device *mdd , + struct dt_key *key, + struct thandle *th) +{ + struct dt_object *dor = mdd->mdd_orphans; + + return dor->do_index_ops->dio_delete(env, dor, + key, th, + BYPASS_CAPA); +} + +static inline void mdd_orphan_ref_add(const struct lu_env *env, + struct mdd_device *mdd, + struct thandle *th) +{ + struct dt_object *dor = mdd->mdd_orphans; + dor->do_ops->do_ref_add(env, dor, th); +} + +static inline void mdd_orphan_ref_del(const struct lu_env *env, + struct mdd_device *mdd, + struct thandle *th) +{ + struct dt_object *dor = mdd->mdd_orphans; + dor->do_ops->do_ref_del(env, dor, th); +} + + +int orph_declare_index_insert(const struct lu_env *env, + struct mdd_object *obj, + umode_t mode, struct thandle *th) +{ + struct dt_insert_rec *rec = &mdd_env_info(env)->mti_dt_rec; + struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); + struct dt_key *key; + int rc; + + key = orph_key_fill(env, mdo2fid(obj), ORPH_OP_UNLINK); + + rec->rec_fid = mdo2fid(obj); + rec->rec_type = mode; + rc = dt_declare_insert(env, mdd->mdd_orphans, + (const struct dt_rec *)rec, key, th); + if (rc != 0) + return rc; + + rc = mdo_declare_ref_add(env, obj, th); + if (rc) + return rc; + + if (!S_ISDIR(mode)) + return 0; + + rc = mdo_declare_ref_add(env, obj, th); + if (rc) + return rc; + + rc = dt_declare_ref_add(env, mdd->mdd_orphans, th); + if (rc) + return rc; + + rc = mdo_declare_index_delete(env, obj, dotdot, th); + if (rc) + return rc; + + rc = mdo_declare_index_insert(env, obj, + lu_object_fid(&mdd->mdd_orphans->do_lu), + S_IFDIR, dotdot, th); + + return rc; +} + +static int orph_index_insert(const struct lu_env *env, + struct mdd_object *obj, + __u32 op, struct thandle *th) +{ + struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); + struct dt_object *dor = mdd->mdd_orphans; + const struct lu_fid *lf_dor = lu_object_fid(&dor->do_lu); + struct dt_object *next = mdd_object_child(obj); + struct dt_insert_rec *rec = &mdd_env_info(env)->mti_dt_rec; + int rc; ENTRY; - rc = dor->do_index_ops->dio_insert(env, dor, (struct dt_rec *)offset, - (struct dt_key *)key, th, - BYPASS_CAPA); + LASSERT(mdd_write_locked(env, obj) != 0); + LASSERT(!(obj->mod_flags & ORPHAN_OBJ)); + + mdd_orphan_write_lock(env, mdd); + + rc = mdd_orphan_insert_obj(env, mdd, obj, op, th); + if (rc) + GOTO(out, rc); + + mdo_ref_add(env, obj, th); + if (!S_ISDIR(mdd_object_type(obj))) + GOTO(out, rc = 0); + + mdo_ref_add(env, obj, th); + mdd_orphan_ref_add(env, mdd, th); + + /* try best to fixup directory, dont return errors + * from here */ + if (!dt_try_as_dir(env, next)) + GOTO(out, rc = 0); + next->do_index_ops->dio_delete(env, next, + (const struct dt_key *)dotdot, + th, BYPASS_CAPA); + + rec->rec_fid = lf_dor; + rec->rec_type = S_IFDIR; + next->do_index_ops->dio_insert(env, next, (const struct dt_rec *)rec, + (const struct dt_key *)dotdot, + th, BYPASS_CAPA, 1); + +out: + if (rc == 0) + obj->mod_flags |= ORPHAN_OBJ; + + mdd_orphan_write_unlock(env, mdd); + RETURN(rc); } +int orph_declare_index_delete(const struct lu_env *env, + struct mdd_object *obj, + + struct thandle *th) +{ + struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); + struct dt_key *key; + int rc; + + key = orph_key_fill(env, mdo2fid(obj), ORPH_OP_UNLINK); + + rc = dt_declare_delete(env, mdd->mdd_orphans, key, th); + if (rc) + return rc; + + rc = mdo_declare_ref_del(env, obj, th); + if (rc) + return rc; + + if (S_ISDIR(mdd_object_type(obj))) { + rc = mdo_declare_ref_del(env, obj, th); + if (rc) + return rc; + + rc = dt_declare_ref_del(env, mdd->mdd_orphans, th); + } + + return rc; +} + static int orph_index_delete(const struct lu_env *env, - struct mdd_object *obj, __u32 op, + struct mdd_object *obj, + __u32 op, struct thandle *th) { struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); struct dt_object *dor = mdd->mdd_orphans; - struct orph_key *key = orph_key_fill(env, mdo2fid(obj), op); + struct dt_key *key; int rc; + ENTRY; + + LASSERT(mdd_write_locked(env, obj) != 0); + LASSERT(obj->mod_flags & ORPHAN_OBJ); + LASSERT(obj->mod_count == 0); + LASSERT(dor); - rc = dor->do_index_ops->dio_delete(env, dor, - (struct dt_key *)key, th, - BYPASS_CAPA); - RETURN(rc); + key = orph_key_fill(env, mdo2fid(obj), op); + mdd_orphan_write_lock(env, mdd); + + rc = mdd_orphan_delete_obj(env, mdd, key, th); + + if (rc == -ENOENT) { + key = orph_key_fill_18(env, mdo2fid(obj)); + rc = mdd_orphan_delete_obj(env, mdd, key, th); + } + + if (!rc) { + /* lov objects will be destroyed by caller */ + mdo_ref_del(env, obj, th); + if (S_ISDIR(mdd_object_type(obj))) { + mdo_ref_del(env, obj, th); + mdd_orphan_ref_del(env, mdd, th); + } + obj->mod_flags &= ~ORPHAN_OBJ; + } else { + CERROR("could not delete object: rc = %d\n",rc); + } + + mdd_orphan_write_unlock(env, mdd); + RETURN(rc); } -static inline struct orph_key *orph_key_empty(const struct lu_env *env, - __u32 op) + +static int orphan_object_destroy(const struct lu_env *env, + struct mdd_object *obj, + struct dt_key *key) { - struct orph_key *key = &mdd_env_info(env)->mti_orph_key; - LASSERT(key); - fid_zero(&key->ok_fid); - key->ok_op = cpu_to_be32(op); - return key; + struct thandle *th = NULL; + struct mdd_device *mdd = mdo2mdd(&obj->mod_obj); + int rc = 0; + ENTRY; + + th = mdd_trans_create(env, mdd); + if (IS_ERR(th)) { + CERROR("Cannot get thandle\n"); + RETURN(PTR_ERR(th)); + } + + rc = orph_declare_index_delete(env, obj, th); + if (rc) + GOTO(stop, rc); + + rc = mdo_declare_destroy(env, obj, th); + if (rc) + GOTO(stop, rc); + + rc = mdd_trans_start(env, mdd, th); + if (rc) + GOTO(stop, rc); + + mdd_write_lock(env, obj, MOR_TGT_CHILD); + if (likely(obj->mod_count == 0)) { + mdd_orphan_write_lock(env, mdd); + rc = mdd_orphan_delete_obj(env, mdd, key, th); + if (rc == 0) { + mdo_ref_del(env, obj, th); + if (S_ISDIR(mdd_object_type(obj))) { + mdo_ref_del(env, obj, th); + mdd_orphan_ref_del(env, mdd, th); + } + rc = mdo_destroy(env, obj, th); + } else + CERROR("could not delete object: rc = %d\n",rc); + mdd_orphan_write_unlock(env, mdd); + } + mdd_write_unlock(env, obj); + +stop: + mdd_trans_stop(env, mdd, 0, th); + + RETURN(rc); } -static void orph_key_test_and_del(const struct lu_env *env, - struct mdd_device *mdd, - const struct orph_key *key) +/** + * Delete unused orphan with FID \a lf from PENDING directory + * + * \param mdd MDD device finishing recovery + * \param lf FID of file or directory to delete + * \param key cookie for this entry in index iterator + * + * \retval 0 success + * \retval -ve error + */ +static int orph_key_test_and_del(const struct lu_env *env, + struct mdd_device *mdd, + struct lu_fid *lf, + struct dt_key *key) { struct mdd_object *mdo; + int rc; + + mdo = mdd_object_find(env, mdd, lf); - mdo = mdd_object_find(env, mdd, &key->ok_fid); if (IS_ERR(mdo)) - CERROR("Invalid orphan!\n"); - else { + return PTR_ERR(mdo); + + rc = -EBUSY; + if (mdo->mod_count == 0) { + CDEBUG(D_HA, "Found orphan "DFID", delete it\n", PFID(lf)); + rc = orphan_object_destroy(env, mdo, key); + if (rc) /* so replay-single.sh test_37 works */ + CERROR("%s: error unlinking orphan "DFID" from " + "PENDING: rc = %d\n", + mdd2obd_dev(mdd)->obd_name, PFID(lf), rc); + } else { mdd_write_lock(env, mdo, MOR_TGT_CHILD); - if (mdo->mod_count == 0) { - /* non-opened orphan, let's delete it */ - struct md_attr *ma = &mdd_env_info(env)->mti_ma; - CWARN("Found orphan!\n"); - mdd_object_kill(env, mdo, ma); - /* TODO: now handle OST objects */ - //mdd_ost_objects_destroy(env, ma); - /* TODO: destroy index entry */ + if (likely(mdo->mod_count > 0)) { + CDEBUG(D_HA, "Found orphan "DFID" count %d, skip it\n", + PFID(lf), mdo->mod_count); + mdo->mod_flags |= ORPHAN_OBJ; } mdd_write_unlock(env, mdo); - mdd_object_put(env, mdo); } + + mdd_object_put(env, mdo); + return rc; } +/** + * delete unreferenced files and directories in the PENDING directory + * + * Files that remain in PENDING after client->MDS recovery has completed + * have to be referenced (opened) by some client during recovery, or they + * will be deleted here (for clients that did not complete recovery). + * + * \param thread info about orphan cleanup thread + * + * \retval 0 success + * \retval -ve error + */ static int orph_index_iterate(const struct lu_env *env, - struct mdd_device *mdd) + struct mdd_generic_thread *thread) { - struct dt_object *dt_obj = mdd->mdd_orphans; - struct dt_it *it; - struct dt_it_ops *iops; - struct orph_key *key = orph_key_empty(env, 0); - int result; + struct mdd_device *mdd = (struct mdd_device *)thread->mgt_data; + struct dt_object *dor = mdd->mdd_orphans; + struct lu_dirent *ent = &mdd_env_info(env)->mti_ent; + const struct dt_it_ops *iops; + struct dt_it *it; + struct lu_fid fid; + int key_sz = 0; + int rc; + __u64 cookie; ENTRY; - iops = &dt_obj->do_index_ops->dio_it; - it = iops->init(env, dt_obj, 1, BYPASS_CAPA); - if (it != NULL) { - result = iops->get(env, it, (const void *)key); - if (result > 0) { - int i; - /* main cycle */ - for (result = 0, i = 0; result == +1; ++i) { - key = (void *)iops->key(env, it); - fid_be_to_cpu(&key->ok_fid, &key->ok_fid); - orph_key_test_and_del(env, mdd, key); - result = iops->next(env, it); - } - } else if (result == 0) - /* Index contains no zero key? */ - result = -EIO; - - iops->put(env, it); - iops->fini(env, it); - } else - result = -ENOMEM; - - RETURN(result); + iops = &dor->do_index_ops->dio_it; + it = iops->init(env, dor, LUDA_64BITHASH, BYPASS_CAPA); + if (IS_ERR(it)) { + rc = PTR_ERR(it); + CERROR("%s: cannot clean PENDING: rc = %d\n", + mdd2obd_dev(mdd)->obd_name, rc); + GOTO(out, rc); + } + + rc = iops->load(env, it, 0); + if (rc < 0) + GOTO(out_put, rc); + if (rc == 0) { + CERROR("%s: error loading iterator to clean PENDING\n", + mdd2obd_dev(mdd)->obd_name); + /* Index contains no zero key? */ + GOTO(out_put, rc = -EIO); + } + + do { + if (thread->mgt_abort) + break; + + key_sz = iops->key_size(env, it); + /* filter out "." and ".." entries from PENDING dir. */ + if (key_sz < 8) + goto next; + + rc = iops->rec(env, it, (struct dt_rec *)ent, LUDA_64BITHASH); + if (rc != 0) { + CERROR("%s: fail to get FID for orphan it: rc = %d\n", + mdd2obd_dev(mdd)->obd_name, rc); + goto next; + } + + fid_le_to_cpu(&fid, &ent->lde_fid); + if (!fid_is_sane(&fid)) { + CERROR("%s: bad FID "DFID" cleaning PENDING\n", + mdd2obd_dev(mdd)->obd_name, PFID(&fid)); + goto next; + } + + /* kill orphan object */ + cookie = iops->store(env, it); + iops->put(env, it); + rc = orph_key_test_and_del(env, mdd, &fid, + (struct dt_key *)ent->lde_name); + + /* after index delete reset iterator */ + if (rc == 0) + rc = iops->get(env, it, (const void *)""); + else + rc = iops->load(env, it, cookie); +next: + rc = iops->next(env, it); + } while (rc == 0); + + GOTO(out_put, rc = 0); +out_put: + iops->put(env, it); + iops->fini(env, it); + +out: + return rc; } +/** + * open the PENDING directory for device \a mdd + * + * The PENDING directory persistently tracks files and directories that were + * unlinked from the namespace (nlink == 0) but are still held open by clients. + * Those inodes shouldn't be deleted if the MDS crashes, because the clients + * would not be able to recover and reopen those files. Instead, these inodes + * are linked into the PENDING directory on disk, and only deleted if all + * clients close them, or the MDS finishes client recovery without any client + * reopening them (i.e. former clients didn't join recovery). + * \param d mdd device being started. + * + * \retval 0 success + * \retval -ve index operation error. + * + */ int orph_index_init(const struct lu_env *env, struct mdd_device *mdd) { - struct lu_fid fid; - struct dt_object *d; - int rc; - ENTRY; + struct lu_fid fid; + struct dt_object *d; + int rc = 0; - d = dt_store_open(env, mdd->mdd_child, orph_index_name, &fid); - if (!IS_ERR(d)) { - mdd->mdd_orphans = d; - rc = d->do_ops->do_index_try(env, d, &orph_index_features); - if (rc == 0) - LASSERT(d->do_index_ops != NULL); - else - CERROR("\"%s\" is not an index!\n", orph_index_name); - } else { - CERROR("cannot find \"%s\" obj %d\n", - orph_index_name, (int)PTR_ERR(d)); - rc = PTR_ERR(d); - } + ENTRY; - RETURN(rc); + /* create PENDING dir */ + fid_zero(&fid); + rc = mdd_local_file_create(env, mdd, &mdd->mdd_local_root_fid, + orph_index_name, S_IFDIR | S_IRUGO | + S_IWUSR | S_IXUGO, &fid); + if (rc < 0) + RETURN(rc); + + d = dt_locate(env, mdd->mdd_child, &fid); + if (IS_ERR(d)) + RETURN(PTR_ERR(d)); + LASSERT(lu_object_exists(&d->do_lu)); + if (!dt_try_as_dir(env, d)) { + CERROR("%s: \"%s\" is not an index: rc = %d\n", + mdd2obd_dev(mdd)->obd_name, orph_index_name, rc); + lu_object_put(env, &d->do_lu); + RETURN(-ENOTDIR); + } + mdd->mdd_orphans = d; + RETURN(0); } void orph_index_fini(const struct lu_env *env, struct mdd_device *mdd) @@ -214,18 +568,87 @@ void orph_index_fini(const struct lu_env *env, struct mdd_device *mdd) EXIT; } -int __mdd_orphan_cleanup(const struct lu_env *env, struct mdd_device *d) +static int __mdd_orphan_cleanup(void *args) { - return orph_index_iterate(env, d); + struct mdd_generic_thread *thread = (struct mdd_generic_thread *)args; + struct lu_env *env = NULL; + int rc; + ENTRY; + + complete(&thread->mgt_started); + + OBD_ALLOC_PTR(env); + if (env == NULL) + GOTO(out, rc = -ENOMEM); + + rc = lu_env_init(env, LCT_MD_THREAD); + if (rc) + GOTO(out, rc); + + rc = orph_index_iterate(env, thread); + + lu_env_fini(env); + GOTO(out, rc); +out: + if (env) + OBD_FREE_PTR(env); + complete(&thread->mgt_finished); + return rc; +} + +/** + * Iterate orphan index to cleanup orphan objects after recovery is done. + * \param d mdd device in recovery. + */ +int mdd_orphan_cleanup(const struct lu_env *env, struct mdd_device *d) +{ + int rc = -ENOMEM; + char *name = NULL; + + OBD_ALLOC(name, MTI_NAME_MAXLEN); + if (name == NULL) + goto out; + + snprintf(name, MTI_NAME_MAXLEN, "orph_cleanup_%s", + mdd2obd_dev(d)->obd_name); + + rc = mdd_generic_thread_start(&d->mdd_orph_cleanup_thread, + __mdd_orphan_cleanup, (void *)d, name); +out: + if (rc) + CERROR("%s: start orphan cleanup thread failed:%d\n", + mdd2obd_dev(d)->obd_name, rc); + if (name) + OBD_FREE(name, MTI_NAME_MAXLEN); + return rc; } +/** + * add an orphan \a obj to the orphan index. + * \param obj file or directory. + * \param th transaction for index insert. + * + * \pre obj nlink == 0 && obj->mod_count != 0 + * + * \retval 0 success + * \retval -ve index operation error. + */ int __mdd_orphan_add(const struct lu_env *env, struct mdd_object *obj, struct thandle *th) { - loff_t offset = 0; - return orph_index_insert(env, obj, ORPH_OP_UNLINK, &offset, th); + return orph_index_insert(env, obj, ORPH_OP_UNLINK, th); } +/** + * delete an orphan \a obj from orphan index. + * \param obj file or directory. + * \param th transaction for index deletion and object destruction. + * + * \pre obj->mod_count == 0 && ORPHAN_OBJ is set for obj. + * + * \retval 0 success + * \retval -ve index operation error. + */ int __mdd_orphan_del(const struct lu_env *env, struct mdd_object *obj, struct thandle *th) {