Whamcloud - gitweb
LU-14098 obdclass: try to skip corrupted llog records
[fs/lustre-release.git] / lustre / obdclass / llog_osd.c
index 6027ef2..63abf66 100644 (file)
@@ -23,7 +23,7 @@
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  *
- * Copyright (c) 2012, 2015, Intel Corporation.
+ * Copyright (c) 2012, 2017, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
@@ -44,6 +44,8 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
+#include <linux/delay.h>
+
 #include <dt_object.h>
 #include <llog_swab.h>
 #include <lustre_fid.h>
@@ -124,8 +126,13 @@ static int llog_osd_create_new_object(const struct lu_env *env,
 static int llog_osd_exist(struct llog_handle *handle)
 {
        LASSERT(handle->lgh_obj);
-       return dt_object_exists(handle->lgh_obj) &&
-               !lu_object_is_dying(handle->lgh_obj->do_lu.lo_header);
+       return dt_object_exists(handle->lgh_obj) && !handle->lgh_destroyed;
+}
+
+static void *rec_tail(struct llog_rec_hdr *rec)
+{
+       return (void *)((char *)rec + rec->lrh_len -
+                       sizeof(struct llog_rec_tail));
 }
 
 /**
@@ -280,7 +287,6 @@ static int llog_osd_read_header(const struct lu_env *env,
 
        handle->lgh_hdr->llh_flags |= (flags & LLOG_F_EXT_MASK);
        handle->lgh_last_idx = LLOG_HDR_TAIL(handle->lgh_hdr)->lrt_index;
-       handle->lgh_write_offset = lgi->lgi_attr.la_size;
 
        RETURN(0);
 }
@@ -357,7 +363,7 @@ static int llog_osd_declare_write_rec(const struct lu_env *env,
  *                             the full llog record to write. This is
  *                             the beginning of buffer to write, the length
  *                             of buffer is stored in \a rec::lrh_len
- * \param[out] reccookie       pointer to the cookie to return back if needed.
+ * \param[in,out] reccookie    pointer to the cookie to return back if needed.
  *                             It is used for further cancel of this llog
  *                             record.
  * \param[in]  idx             index of the llog record. If \a idx == -1 then
@@ -384,7 +390,6 @@ static int llog_osd_write_rec(const struct lu_env *env,
        __u32                   chunk_size;
        size_t                   left;
        __u32                   orig_last_idx;
-       __u64                   orig_write_offset;
        ENTRY;
 
        llh = loghandle->lgh_hdr;
@@ -432,7 +437,7 @@ static int llog_osd_write_rec(const struct lu_env *env,
                /* llog can be empty only when first record is being written */
                LASSERT(ergo(idx > 0, lgi->lgi_attr.la_size > 0));
 
-               if (!ext2_test_bit(idx, LLOG_HDR_BITMAP(llh))) {
+               if (!test_bit_le(idx, LLOG_HDR_BITMAP(llh))) {
                        CERROR("%s: modify unset record %u\n",
                               o->do_lu.lo_dev->ld_obd->obd_name, idx);
                        RETURN(-ENOENT);
@@ -486,26 +491,26 @@ static int llog_osd_write_rec(const struct lu_env *env,
                                             &lgi->lgi_off, th);
 
                        RETURN(rc);
-               } else if (loghandle->lgh_cur_idx > 0) {
+               } else if (llh->llh_flags & LLOG_F_IS_FIXSIZE) {
+                       lgi->lgi_off = llh->llh_hdr.lrh_len +
+                                      (idx - 1) * reclen;
+               } else if (reccookie != NULL && reccookie->lgc_index > 0) {
                        /**
-                        * The lgh_cur_offset can be used only if index is
+                        * The lgc_offset can be used only if index is
                         * the same.
                         */
-                       if (idx != loghandle->lgh_cur_idx) {
+                       if (idx != reccookie->lgc_index) {
                                CERROR("%s: modify index mismatch %d %d\n",
                                       o->do_lu.lo_dev->ld_obd->obd_name, idx,
-                                      loghandle->lgh_cur_idx);
+                                      reccookie->lgc_index);
                                RETURN(-EFAULT);
                        }
 
-                       lgi->lgi_off = loghandle->lgh_cur_offset;
-                       CDEBUG(D_OTHER, "modify record "DOSTID": idx:%d, "
+                       lgi->lgi_off = reccookie->lgc_offset;
+                       CDEBUG(D_OTHER, "modify record "DFID": idx:%u, "
                               "len:%u offset %llu\n",
-                              POSTID(&loghandle->lgh_id.lgl_oi), idx,
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid), idx,
                               rec->lrh_len, (long long)lgi->lgi_off);
-               } else if (llh->llh_flags & LLOG_F_IS_FIXSIZE) {
-                       lgi->lgi_off = llh->llh_hdr.lrh_len +
-                                      (idx - 1) * reclen;
                } else {
                        /* This can be result of lgh_cur_idx is not set during
                         * llog processing or llh_size is not set to proper
@@ -550,8 +555,19 @@ static int llog_osd_write_rec(const struct lu_env *env,
 
        LASSERT(lgi->lgi_attr.la_valid & LA_SIZE);
        orig_last_idx = loghandle->lgh_last_idx;
-       orig_write_offset = loghandle->lgh_write_offset;
        lgi->lgi_off = lgi->lgi_attr.la_size;
+
+       if (loghandle->lgh_max_size > 0 &&
+           lgi->lgi_off >= loghandle->lgh_max_size) {
+               CDEBUG(D_OTHER, "llog is getting too large (%u > %u) at %u "
+                      DFID"\n", (unsigned)lgi->lgi_off,
+                      loghandle->lgh_max_size, (int)loghandle->lgh_last_idx,
+                      PFID(&loghandle->lgh_id.lgl_oi.oi_fid));
+               /* this is to signal that this llog is full */
+               loghandle->lgh_last_idx = LLOG_HDR_BITMAP_SIZE(llh) - 1;
+               RETURN(-ENOSPC);
+       }
+
        left = chunk_size - (lgi->lgi_off & (chunk_size - 1));
        /* NOTE: padding is a record, but no bit is set */
        if (left != 0 && left != reclen &&
@@ -561,9 +577,6 @@ static int llog_osd_write_rec(const struct lu_env *env,
                if (rc)
                        RETURN(rc);
 
-               if (dt_object_remote(o))
-                       loghandle->lgh_write_offset = lgi->lgi_off;
-
                loghandle->lgh_last_idx++; /* for pad rec */
        }
        /* if it's the last idx in log file, then return -ENOSPC
@@ -578,6 +591,7 @@ static int llog_osd_write_rec(const struct lu_env *env,
                        RETURN(-ENOSPC);
        }
 
+       down_write(&loghandle->lgh_last_sem);
        /* increment the last_idx along with llh_tail index, they should
         * be equal for a llog lifetime */
        loghandle->lgh_last_idx++;
@@ -597,9 +611,10 @@ static int llog_osd_write_rec(const struct lu_env *env,
        /* the lgh_hdr_mutex protects llog header data from concurrent
         * update/cancel, the llh_count and llh_bitmap are protected */
        mutex_lock(&loghandle->lgh_hdr_mutex);
-       if (ext2_set_bit(index, LLOG_HDR_BITMAP(llh))) {
-               CERROR("%s: index %u already set in log bitmap\n",
-                      o->do_lu.lo_dev->ld_obd->obd_name, index);
+       if (__test_and_set_bit_le(index, LLOG_HDR_BITMAP(llh))) {
+               CERROR("%s: index %u already set in llog bitmap "DFID"\n",
+                      o->do_lu.lo_dev->ld_obd->obd_name, index,
+                      PFID(lu_object_fid(&o->do_lu)));
                mutex_unlock(&loghandle->lgh_hdr_mutex);
                LBUG(); /* should never happen */
        }
@@ -661,13 +676,16 @@ out_unlock:
        if (rc)
                GOTO(out, rc);
 
+       if (OBD_FAIL_PRECHECK(OBD_FAIL_LLOG_PROCESS_TIMEOUT) &&
+          cfs_fail_val == (unsigned int)(loghandle->lgh_id.lgl_oi.oi.oi_id &
+                                         0xFFFFFFFF)) {
+               OBD_RACE(OBD_FAIL_LLOG_PROCESS_TIMEOUT);
+               msleep(1 * MSEC_PER_SEC);
+       }
        /* computed index can be used to determine offset for fixed-size
         * records. This also allows to handle Catalog wrap around case */
        if (llh->llh_flags & LLOG_F_IS_FIXSIZE) {
                lgi->lgi_off = llh->llh_hdr.lrh_len + (index - 1) * reclen;
-       } else if (dt_object_remote(o)) {
-               lgi->lgi_off = max_t(__u64, loghandle->lgh_write_offset,
-                                    lgi->lgi_off);
        } else {
                rc = dt_attr_get(env, o, &lgi->lgi_attr);
                if (rc)
@@ -684,10 +702,9 @@ out_unlock:
        if (rc < 0)
                GOTO(out, rc);
 
-       if (dt_object_remote(o))
-               loghandle->lgh_write_offset = lgi->lgi_off;
+       up_write(&loghandle->lgh_last_sem);
 
-       CDEBUG(D_HA, "added record "DFID": idx: %u, %u off"LPU64"\n",
+       CDEBUG(D_HA, "added record "DFID".%u, %u off%llu\n",
               PFID(lu_object_fid(&o->do_lu)), index, rec->lrh_len,
               lgi->lgi_off);
        if (reccookie != NULL) {
@@ -706,14 +723,13 @@ out_unlock:
 out:
        /* cleanup llog for error case */
        mutex_lock(&loghandle->lgh_hdr_mutex);
-       ext2_clear_bit(index, LLOG_HDR_BITMAP(llh));
+       clear_bit_le(index, LLOG_HDR_BITMAP(llh));
        llh->llh_count--;
        mutex_unlock(&loghandle->lgh_hdr_mutex);
 
        /* restore llog last_idx */
        if (dt_object_remote(o)) {
                loghandle->lgh_last_idx = orig_last_idx;
-               loghandle->lgh_write_offset = orig_write_offset;
        } else if (--loghandle->lgh_last_idx == 0 &&
            (llh->llh_flags & LLOG_F_IS_CAT) && llh->llh_cat_idx != 0) {
                /* catalog had just wrap-around case */
@@ -721,6 +737,7 @@ out:
        }
 
        LLOG_HDR_TAIL(llh)->lrt_index = loghandle->lgh_last_idx;
+       up_write(&loghandle->lgh_last_sem);
 
        RETURN(rc);
 }
@@ -776,22 +793,67 @@ static inline void llog_skip_over(struct llog_handle *lgh, __u64 *off,
  * big enough to handle the remapped records. It is also assumed that records
  * of a block have the same format (i.e.: the same features enabled).
  *
- * \param[in,out]    hdr       Header of the block of records to remap.
- * \param[in,out]    last_hdr   Last header, don't read past this point.
- * \param[in]        flags     Flags describing the fields to keep.
+ * \param[in,out]    hdr          Header of the block of records to remap.
+ * \param[in,out]    last_hdr      Last header, don't read past this point.
+ * \param[in]        flags        Flags describing the fields to keep.
+ * \param[in]        extra_flags   Flags describing the extra fields to keep.
  */
 static void changelog_block_trim_ext(struct llog_rec_hdr *hdr,
                                     struct llog_rec_hdr *last_hdr,
-                                    enum changelog_rec_flags flags)
+                                    struct llog_handle *loghandle)
 {
+       enum changelog_rec_flags flags = CLF_SUPPORTED;
+       enum changelog_rec_extra_flags extra_flags = CLFE_SUPPORTED;
+
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_XATTR))
+               extra_flags &= ~CLFE_XATTR;
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_OMODE))
+               extra_flags &= ~CLFE_OPEN;
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_NID))
+               extra_flags &= ~CLFE_NID;
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_X_UIDGID))
+               extra_flags &= ~CLFE_UIDGID;
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_EXTRA_FLAGS))
+               flags &= ~CLF_EXTRA_FLAGS;
+       if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID))
+               flags &= ~CLF_JOBID;
+
+       if (flags == CLF_SUPPORTED && extra_flags == CLFE_SUPPORTED)
+               return;
+
        if (hdr->lrh_type != CHANGELOG_REC)
                return;
 
        do {
                struct changelog_rec *rec = (struct changelog_rec *)(hdr + 1);
+               enum changelog_rec_extra_flags xflag = CLFE_INVALID;
+
+               if (flags & CLF_EXTRA_FLAGS &&
+                   rec->cr_flags & CLF_EXTRA_FLAGS) {
+                       xflag = changelog_rec_extra_flags(rec)->cr_extra_flags &
+                               extra_flags;
+               }
+
+               if (unlikely(hdr->lrh_len == 0)) {
+                       /* It is corruption case, we cannot know the next rec,
+                        * jump to the last one directly to avoid dead loop. */
+                       LCONSOLE(D_WARNING, "Hit invalid llog record: "
+                                "idx %u, type %u, id %u\n",
+                                hdr->lrh_index, hdr->lrh_type, hdr->lrh_id);
+                       hdr = llog_rec_hdr_next(last_hdr);
+                       if (unlikely(hdr == last_hdr))
+                               LCONSOLE(D_WARNING, "The last record crashed: "
+                                        "idx %u, type %u, id %u\n",
+                                        hdr->lrh_index, hdr->lrh_type,
+                                        hdr->lrh_id);
+                       break;
+               }
 
-               changelog_remap_rec(rec, rec->cr_flags & flags);
+               changelog_remap_rec(rec, rec->cr_flags & flags, xflag);
                hdr = llog_rec_hdr_next(hdr);
+               /* Yield CPU to avoid soft-lockup if there are too many records
+                * to be handled. */
+               cond_resched();
        } while ((char *)hdr <= (char *)last_hdr);
 }
 
@@ -841,7 +903,7 @@ static int llog_osd_next_block(const struct lu_env *env,
 
        o = loghandle->lgh_obj;
        LASSERT(o);
-       LASSERT(dt_object_exists(o));
+       LASSERT(llog_osd_exist(loghandle));
        dt = lu2dt_dev(o->do_lu.lo_dev);
        LASSERT(dt);
 
@@ -849,8 +911,9 @@ static int llog_osd_next_block(const struct lu_env *env,
        if (rc)
                GOTO(out, rc);
 
-       CDEBUG(D_OTHER, "looking for log index %u (cur idx %u off"
-              LPU64"), size %llu\n", next_idx, *cur_idx,
+       CDEBUG(D_OTHER,
+              "looking for log index %u (cur idx %u off %llu), size %llu\n",
+              next_idx, *cur_idx,
               *cur_offset, lgi->lgi_attr.la_size);
 
        while (*cur_offset < lgi->lgi_attr.la_size) {
@@ -871,7 +934,7 @@ static int llog_osd_next_block(const struct lu_env *env,
                                goto retry;
 
                        CERROR("%s: can't read llog block from log "DFID
-                              " offset "LPU64": rc = %d\n",
+                              " offset %llu: rc = %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
                               PFID(lu_object_fid(&o->do_lu)), *cur_offset,
                               rc);
@@ -894,10 +957,10 @@ static int llog_osd_next_block(const struct lu_env *env,
                        if (!force_mini_rec)
                                goto retry;
 
-                       CERROR("%s: invalid llog block at log id "DOSTID"/%u "
-                              "offset "LPU64"\n",
+                       CERROR("%s: invalid llog block at log id "DFID":%x "
+                              "offset %llu\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
-                              POSTID(&loghandle->lgh_id.lgl_oi),
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid),
                               loghandle->lgh_id.lgl_ogen, *cur_offset);
                        GOTO(out, rc = -EINVAL);
                }
@@ -905,9 +968,25 @@ static int llog_osd_next_block(const struct lu_env *env,
                rec = buf;
                if (LLOG_REC_HDR_NEEDS_SWABBING(rec))
                        lustre_swab_llog_rec(rec);
-
                tail = (struct llog_rec_tail *)((char *)buf + rc -
                                                sizeof(struct llog_rec_tail));
+
+               if (llog_verify_record(loghandle, rec)) {
+                       /*
+                        * the block seems corrupted. make a pad record so the
+                        * caller can skip the block and try with the next one
+                        */
+                       rec->lrh_len = rc;
+                       rec->lrh_index = next_idx;
+                       rec->lrh_type = LLOG_PAD_MAGIC;
+
+                       tail = rec_tail(rec);
+                       tail->lrt_len = rc;
+                       tail->lrt_index = next_idx;
+
+                       GOTO(out, rc = 0);
+               }
+
                /* get the last record in block */
                last_rec = (struct llog_rec_hdr *)((char *)buf + rc -
                                                   tail->lrt_len);
@@ -916,12 +995,12 @@ static int llog_osd_next_block(const struct lu_env *env,
                        lustre_swab_llog_rec(last_rec);
 
                if (last_rec->lrh_index != tail->lrt_index) {
-                       CERROR("%s: invalid llog tail at log id "DOSTID"/%u "
-                              "offset "LPU64" last_rec idx %u tail idx %u\n",
+                       CERROR("%s: invalid llog tail at log id "DFID":%x offset %llu last_rec idx %u tail idx %u lrt len %u read_size %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
-                              POSTID(&loghandle->lgh_id.lgl_oi),
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid),
                               loghandle->lgh_id.lgl_ogen, *cur_offset,
-                              last_rec->lrh_index, tail->lrt_index);
+                              last_rec->lrh_index, tail->lrt_index,
+                              tail->lrt_len, rc);
                        GOTO(out, rc = -EINVAL);
                }
 
@@ -929,10 +1008,10 @@ static int llog_osd_next_block(const struct lu_env *env,
 
                /* this shouldn't happen */
                if (tail->lrt_index == 0) {
-                       CERROR("%s: invalid llog tail at log id "DOSTID"/%u "
-                              "offset "LPU64" bytes %d\n",
+                       CERROR("%s: invalid llog tail at log id "DFID":%x "
+                              "offset %llu bytes %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
-                              POSTID(&loghandle->lgh_id.lgl_oi),
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid),
                               loghandle->lgh_id.lgl_ogen, *cur_offset, rc);
                        GOTO(out, rc = -EINVAL);
                }
@@ -944,7 +1023,7 @@ static int llog_osd_next_block(const struct lu_env *env,
 
                /* sanity check that the start of the new buffer is no farther
                 * than the record that we wanted.  This shouldn't happen. */
-               if (rec->lrh_index > next_idx) {
+               if (next_idx && rec->lrh_index > next_idx) {
                        if (!force_mini_rec && next_idx > last_idx)
                                goto retry;
 
@@ -955,9 +1034,7 @@ static int llog_osd_next_block(const struct lu_env *env,
                }
 
                /* Trim unsupported extensions for compat w/ older clients */
-               if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID))
-                       changelog_block_trim_ext(rec, last_rec,
-                                                CLF_VERSION | CLF_RENAME);
+               changelog_block_trim_ext(rec, last_rec, loghandle);
 
                GOTO(out, rc = 0);
 
@@ -1015,7 +1092,7 @@ static int llog_osd_prev_block(const struct lu_env *env,
 
        o = loghandle->lgh_obj;
        LASSERT(o);
-       LASSERT(dt_object_exists(o));
+       LASSERT(llog_osd_exist(loghandle));
        dt = lu2dt_dev(o->do_lu.lo_dev);
        LASSERT(dt);
 
@@ -1038,7 +1115,7 @@ static int llog_osd_prev_block(const struct lu_env *env,
                rc = dt_read(env, o, &lgi->lgi_buf, &cur_offset);
                if (rc < 0) {
                        CERROR("%s: can't read llog block from log "DFID
-                              " offset "LPU64": rc = %d\n",
+                              " offset %llu: rc = %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
                               PFID(lu_object_fid(&o->do_lu)), cur_offset, rc);
                        GOTO(out, rc);
@@ -1048,10 +1125,10 @@ static int llog_osd_prev_block(const struct lu_env *env,
                        GOTO(out, rc);
 
                if (rc < sizeof(*tail)) {
-                       CERROR("%s: invalid llog block at log id "DOSTID"/%u "
-                              "offset "LPU64"\n",
+                       CERROR("%s: invalid llog block at log id "DFID":%x "
+                              "offset %llu\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
-                              POSTID(&loghandle->lgh_id.lgl_oi),
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid),
                               loghandle->lgh_id.lgl_ogen, cur_offset);
                        GOTO(out, rc = -EINVAL);
                }
@@ -1072,10 +1149,10 @@ static int llog_osd_prev_block(const struct lu_env *env,
 
                /* this shouldn't happen */
                if (tail->lrt_index == 0) {
-                       CERROR("%s: invalid llog tail at log id "DOSTID"/%u "
-                              "offset "LPU64"\n",
+                       CERROR("%s: invalid llog tail at log id "DFID":%x "
+                              "offset %llu\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
-                              POSTID(&loghandle->lgh_id.lgl_oi),
+                              PFID(&loghandle->lgh_id.lgl_oi.oi_fid),
                               loghandle->lgh_id.lgl_ogen, cur_offset);
                        GOTO(out, rc = -EINVAL);
                }
@@ -1092,9 +1169,7 @@ static int llog_osd_prev_block(const struct lu_env *env,
                }
 
                /* Trim unsupported extensions for compat w/ older clients */
-               if (!(loghandle->lgh_hdr->llh_flags & LLOG_F_EXT_JOBID))
-                       changelog_block_trim_ext(rec, last_rec,
-                                                CLF_VERSION | CLF_RENAME);
+               changelog_block_trim_ext(rec, last_rec, loghandle);
 
                GOTO(out, rc = 0);
        }
@@ -1129,7 +1204,7 @@ static struct dt_object *llog_osd_dir_get(const struct lu_env *env,
                dir = dt_locate(env, dt, &dti->dti_fid);
 
                if (!IS_ERR(dir) && !dt_try_as_dir(env, dir)) {
-                       lu_object_put(env, &dir->do_lu);
+                       dt_object_put(env, dir);
                        return ERR_PTR(-ENOTDIR);
                }
        } else {
@@ -1189,8 +1264,7 @@ static int llog_osd_open(const struct lu_env *env, struct llog_handle *handle,
                } else {
                        /* If logid == NULL, then it means the caller needs
                         * to allocate new FID (llog_cat_declare_add_rec()). */
-                       rc = obd_fid_alloc(env, ctxt->loc_exp,
-                                          &lgi->lgi_fid, NULL);
+                       rc = dt_fid_alloc(env, dt, &lgi->lgi_fid, NULL, NULL);
                        if (rc < 0)
                                RETURN(rc);
                        rc = 0;
@@ -1228,7 +1302,7 @@ static int llog_osd_open(const struct lu_env *env, struct llog_handle *handle,
                dt_read_lock(env, llog_dir, 0);
                rc = dt_lookup_dir(env, llog_dir, name, &lgi->lgi_fid);
                dt_read_unlock(env, llog_dir);
-               lu_object_put(env, &llog_dir->do_lu);
+               dt_object_put(env, llog_dir);
                if (rc == -ENOENT && open_param == LLOG_OPEN_NEW) {
                        /* generate fid for new llog */
                        rc = local_object_fid_generate(env, los,
@@ -1251,7 +1325,12 @@ generate:
                        GOTO(out, rc);
                new_id = true;
        }
-
+       if (OBD_FAIL_PRECHECK(OBD_FAIL_MDS_LLOG_UMOUNT_RACE) &&
+           cfs_fail_val == 1) {
+               cfs_fail_val = 2;
+               OBD_RACE(OBD_FAIL_MDS_LLOG_UMOUNT_RACE);
+               msleep(MSEC_PER_SEC);
+       }
        o = ls_locate(env, ls, &lgi->lgi_fid, NULL);
        if (IS_ERR(o))
                GOTO(out_name, rc = PTR_ERR(o));
@@ -1263,7 +1342,7 @@ generate:
                       ", skipping\n",
                       o->do_lu.lo_dev->ld_obd->obd_name,
                       PFID(lu_object_fid(&o->do_lu)));
-               lu_object_put(env, &o->do_lu);
+               dt_object_put(env, o);
                /* just skip this llog ID, we shouldn't delete it because we
                 * don't know exactly what is its purpose and state. */
                goto generate;
@@ -1271,9 +1350,12 @@ generate:
 
 after_open:
        /* No new llog is expected but doesn't exist */
-       if (open_param != LLOG_OPEN_NEW && !dt_object_exists(o))
+       if (open_param != LLOG_OPEN_NEW && !dt_object_exists(o)) {
+               CDEBUG(D_INFO, "%s: llog FID: "DFID" obj %p doesn`t exist\n",
+                      o->do_lu.lo_dev->ld_obd->obd_name,
+                      PFID(lu_object_fid(&o->do_lu)), o);
                GOTO(out_put, rc = -ENOENT);
-
+       }
        fid_to_logid(&lgi->lgi_fid, &handle->lgh_id);
        handle->lgh_obj = o;
        handle->private_data = los;
@@ -1282,7 +1364,7 @@ after_open:
        RETURN(rc);
 
 out_put:
-       lu_object_put(env, &o->do_lu);
+       dt_object_put(env, o);
 out_name:
        if (handle->lgh_name != NULL)
                OBD_FREE(handle->lgh_name, strlen(name) + 1);
@@ -1329,7 +1411,7 @@ struct dt_object *llog_osd_get_regular_fid_dir(const struct lu_env *env,
                RETURN(dir);
 
        if (!dt_try_as_dir(env, dir)) {
-               lu_object_put(env, &dir->do_lu);
+               dt_object_put(env, dir);
                RETURN(ERR_PTR(-ENOTDIR));
        }
 
@@ -1380,11 +1462,11 @@ llog_osd_regular_fid_add_name_entry(const struct lu_env *env,
                               (struct dt_key *)name, th);
        } else {
                rc = dt_insert(env, dir, (struct dt_rec *)rec,
-                              (struct dt_key *)name, th, 1);
+                              (struct dt_key *)name, th);
        }
        dt_write_unlock(env, dir);
 
-       lu_object_put(env, &dir->do_lu);
+       dt_object_put(env, dir);
        RETURN(rc);
 }
 
@@ -1461,7 +1543,7 @@ static int llog_osd_declare_create(const struct lu_env *env,
                rc = dt_declare_insert(env, llog_dir,
                                       (struct dt_rec *)rec,
                                       (struct dt_key *)res->lgh_name, th);
-               lu_object_put(env, &llog_dir->do_lu);
+               dt_object_put(env, llog_dir);
                if (rc)
                        CERROR("%s: can't declare named llog %s: rc = %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
@@ -1547,10 +1629,9 @@ static int llog_osd_create(const struct lu_env *env, struct llog_handle *res,
                rec->rec_type = S_IFREG;
                dt_read_lock(env, llog_dir, 0);
                rc = dt_insert(env, llog_dir, (struct dt_rec *)rec,
-                              (struct dt_key *)res->lgh_name,
-                              th, 1);
+                              (struct dt_key *)res->lgh_name, th);
                dt_read_unlock(env, llog_dir);
-               lu_object_put(env, &llog_dir->do_lu);
+               dt_object_put(env, llog_dir);
                if (rc)
                        CERROR("%s: can't create named llog %s: rc = %d\n",
                               o->do_lu.lo_dev->ld_obd->obd_name,
@@ -1583,11 +1664,11 @@ static int llog_osd_close(const struct lu_env *env, struct llog_handle *handle)
        if (handle->lgh_ctxt->loc_flags & LLOG_CTXT_FLAG_NORMAL_FID) {
                /* Remove the object from the cache, otherwise it may
                 * hold LOD being released during cleanup process */
-               lu_object_put_nocache(env, &handle->lgh_obj->do_lu);
+               dt_object_put_nocache(env, handle->lgh_obj);
                LASSERT(handle->private_data == NULL);
                RETURN(rc);
        } else {
-               lu_object_put(env, &handle->lgh_obj->do_lu);
+               dt_object_put(env, handle->lgh_obj);
        }
        los = handle->private_data;
        LASSERT(los);
@@ -1642,7 +1723,7 @@ llog_osd_regular_fid_del_name_entry(const struct lu_env *env,
        }
        dt_write_unlock(env, dir);
 
-       lu_object_put(env, &dir->do_lu);
+       dt_object_put(env, dir);
        RETURN(rc);
 }
 
@@ -1702,7 +1783,7 @@ static int llog_osd_declare_destroy(const struct lu_env *env,
 
 out_put:
        if (!(IS_ERR_OR_NULL(llog_dir)))
-               lu_object_put(env, &llog_dir->do_lu);
+               dt_object_put(env, llog_dir);
 
        RETURN(rc);
 }
@@ -1738,7 +1819,7 @@ static int llog_osd_destroy(const struct lu_env *env,
        LASSERT(o != NULL);
 
        dt_write_lock(env, o, 0);
-       if (!dt_object_exists(o))
+       if (!llog_osd_exist(loghandle))
                GOTO(out_unlock, rc = 0);
 
        if (loghandle->lgh_name) {
@@ -1764,6 +1845,7 @@ static int llog_osd_destroy(const struct lu_env *env,
        if (rc < 0)
                GOTO(out_unlock, rc);
 
+       loghandle->lgh_destroyed = true;
        if (loghandle->lgh_ctxt->loc_flags & LLOG_CTXT_FLAG_NORMAL_FID) {
                rc = llog_osd_regular_fid_del_name_entry(env, o, th, false);
                if (rc < 0)
@@ -1773,7 +1855,7 @@ static int llog_osd_destroy(const struct lu_env *env,
 out_unlock:
        dt_write_unlock(env, o);
        if (!(IS_ERR_OR_NULL(llog_dir)))
-               lu_object_put(env, &llog_dir->do_lu);
+               dt_object_put(env, llog_dir);
        RETURN(rc);
 }
 
@@ -1949,7 +2031,7 @@ int llog_osd_get_cat_list(const struct lu_env *env, struct dt_device *d,
                if (IS_ERR(th))
                        GOTO(out, rc = PTR_ERR(th));
 
-               lgi->lgi_attr.la_valid = LA_MODE;
+               lgi->lgi_attr.la_valid = LA_MODE | LA_TYPE;
                lgi->lgi_attr.la_mode = S_IFREG | S_IRUGO | S_IWUSR;
                lgi->lgi_dof.dof_type = dt_mode_to_dft(S_IFREG);
 
@@ -2019,7 +2101,7 @@ out_trans:
 
        EXIT;
 out:
-       lu_object_put(env, &o->do_lu);
+       dt_object_put(env, o);
        RETURN(rc);
 }
 EXPORT_SYMBOL(llog_osd_get_cat_list);
@@ -2109,7 +2191,7 @@ int llog_osd_put_cat_list(const struct lu_env *env, struct dt_device *d,
 out_trans:
        dt_trans_stop(env, d, th);
 out:
-       lu_object_put(env, &o->do_lu);
+       dt_object_put(env, o);
        RETURN(rc);
 }
 EXPORT_SYMBOL(llog_osd_put_cat_list);