Whamcloud - gitweb
LU-6875 update: set st to NULL in error handler
[fs/lustre-release.git] / lustre / target / update_trans.c
index 7ff50cf..00bc034 100644 (file)
@@ -81,12 +81,17 @@ static void top_multiple_thandle_dump(struct top_multiple_thandle *tmt,
               tmt->tmt_result, tmt->tmt_batchid);
 
        list_for_each_entry(st, &tmt->tmt_sub_thandle_list, st_sub_list) {
-               CDEBUG(mask, "st %p obd %s committed %d sub_th %p "
-                      " cookie "DOSTID": %u\n",
+               struct sub_thandle_cookie *stc;
+
+               CDEBUG(mask, "st %p obd %s committed %d sub_th %p\n",
                       st, st->st_dt->dd_lu_dev.ld_obd->obd_name,
-                      st->st_committed, st->st_sub_th,
-                      POSTID(&st->st_cookie.lgc_lgl.lgl_oi),
-                      st->st_cookie.lgc_index);
+                      st->st_committed, st->st_sub_th);
+
+               list_for_each_entry(stc, &st->st_cookie_list, stc_list) {
+                       CDEBUG(mask, " cookie "DOSTID": %u\n",
+                              POSTID(&stc->stc_cookie.lgc_lgl.lgl_oi),
+                              stc->stc_cookie.lgc_index);
+               }
        }
 }
 
@@ -99,33 +104,46 @@ static void top_multiple_thandle_dump(struct top_multiple_thandle *tmt,
  * \param[in] env      execution environment
  * \param[in] record   update records being written
  * \param[in] sub_th   sub transaction handle
+ * \param[in] record_size total update record size
  *
  * \retval             0 if writing succeeds
  * \retval             negative errno if writing fails
  */
 static int sub_declare_updates_write(const struct lu_env *env,
                                     struct llog_update_record *record,
-                                    struct thandle *sub_th)
+                                    struct thandle *sub_th, size_t record_size)
 {
        struct llog_ctxt        *ctxt;
        struct dt_device        *dt = sub_th->th_dev;
+       int                     left = record_size;
        int rc;
 
        /* If ctxt is NULL, it means not need to write update,
         * for example if the the OSP is used to connect to OST */
        ctxt = llog_get_context(dt->dd_lu_dev.ld_obd,
                                LLOG_UPDATELOG_ORIG_CTXT);
-       LASSERT(ctxt != NULL);
 
        /* Not ready to record updates yet. */
-       if (ctxt->loc_handle == NULL) {
+       if (ctxt == NULL || ctxt->loc_handle == NULL) {
                llog_ctxt_put(ctxt);
                return 0;
        }
 
-       rc = llog_declare_add(env, ctxt->loc_handle, &record->lur_hdr,
-                             sub_th);
+       rc = llog_declare_add(env, ctxt->loc_handle,
+                             &record->lur_hdr, sub_th);
+       if (rc < 0)
+               GOTO(out_put, rc);
 
+       while (left > ctxt->loc_chunk_size) {
+               rc = llog_declare_add(env, ctxt->loc_handle,
+                                     &record->lur_hdr, sub_th);
+               if (rc < 0)
+                       GOTO(out_put, rc);
+
+               left -= ctxt->loc_chunk_size;
+       }
+
+out_put:
        llog_ctxt_put(ctxt);
 
        return rc;
@@ -148,21 +166,30 @@ static int sub_declare_updates_write(const struct lu_env *env,
  */
 static int sub_updates_write(const struct lu_env *env,
                             struct llog_update_record *record,
-                            struct thandle *sub_th,
-                            struct llog_cookie *cookie)
+                            struct sub_thandle *sub_th)
 {
-       struct dt_device        *dt = sub_th->th_dev;
+       struct dt_device        *dt = sub_th->st_dt;
        struct llog_ctxt        *ctxt;
        int                     rc;
+       struct llog_update_record *lur = NULL;
+       struct update_params    *params = NULL;
+       __u32                   update_count = 0;
+       __u32                   param_count = 0;
+       __u32                   last_update_count = 0;
+       __u32                   last_param_count = 0;
+       void                    *src;
+       void                    *start;
+       void                    *next;
+       struct sub_thandle_cookie *stc;
        ENTRY;
 
        ctxt = llog_get_context(dt->dd_lu_dev.ld_obd,
                                LLOG_UPDATELOG_ORIG_CTXT);
-       LASSERT(ctxt != NULL);
-
-       /* Not ready to record updates yet, usually happens
-        * in error handler path */
-       if (ctxt->loc_handle == NULL) {
+       /* If ctxt == NULL, then it means updates on OST (only happens
+        * during migration), and we do not track those updates for now */
+       /* If ctxt->loc_handle == NULL, then it does not need to record
+        * update, usually happens in error handler path */
+       if (ctxt == NULL || ctxt->loc_handle == NULL) {
                llog_ctxt_put(ctxt);
                RETURN(0);
        }
@@ -174,16 +201,124 @@ static int sub_updates_write(const struct lu_env *env,
                 "lrh_len %u record_size %zu\n", record->lur_hdr.lrh_len,
                 llog_update_record_size(record));
 
-       rc = llog_add(env, ctxt->loc_handle, &record->lur_hdr,
-                     cookie, sub_th);
-       llog_ctxt_put(ctxt);
+       if (likely(record->lur_hdr.lrh_len <= ctxt->loc_chunk_size)) {
+               OBD_ALLOC_PTR(stc);
+               if (stc == NULL)
+                       GOTO(llog_put, rc = -ENOMEM);
+               INIT_LIST_HEAD(&stc->stc_list);
+
+               rc = llog_add(env, ctxt->loc_handle, &record->lur_hdr,
+                             &stc->stc_cookie, sub_th->st_sub_th);
+
+               CDEBUG(D_INFO, "%s: Add update log "DOSTID":%u: rc = %d\n",
+                      dt->dd_lu_dev.ld_obd->obd_name,
+                      POSTID(&stc->stc_cookie.lgc_lgl.lgl_oi),
+                      stc->stc_cookie.lgc_index, rc);
+
+               if (rc > 0) {
+                       list_add(&stc->stc_list, &sub_th->st_cookie_list);
+                       rc = 0;
+               } else {
+                       OBD_FREE_PTR(stc);
+               }
 
-       CDEBUG(D_INFO, "%s: Add update log "DOSTID":%u.\n",
-              dt->dd_lu_dev.ld_obd->obd_name,
-              POSTID(&cookie->lgc_lgl.lgl_oi), cookie->lgc_index);
+               GOTO(llog_put, rc);
+       }
 
-       if (rc > 0)
-               rc = 0;
+       /* Split the records into chunk_size update record */
+       OBD_ALLOC_LARGE(lur, ctxt->loc_chunk_size);
+       if (lur == NULL)
+               GOTO(llog_put, rc = -ENOMEM);
+
+       memcpy(lur, &record->lur_hdr, sizeof(record->lur_hdr));
+       lur->lur_update_rec.ur_update_count = 0;
+       lur->lur_update_rec.ur_param_count = 0;
+       src = &record->lur_update_rec.ur_ops;
+       start = next = src;
+       lur->lur_hdr.lrh_len = llog_update_record_size(lur);
+       params = update_records_get_params(&record->lur_update_rec);
+       do {
+               size_t rec_len;
+
+               if (update_count < record->lur_update_rec.ur_update_count) {
+                       next = update_op_next_op((struct update_op *)src);
+               } else {
+                       if (param_count == 0)
+                               next = update_records_get_params(
+                                               &record->lur_update_rec);
+                       else
+                               next = (char *)src +
+                                       object_update_param_size(
+                                       (struct object_update_param *)src);
+               }
+
+               rec_len = cfs_size_round((unsigned long)(next - src));
+               /* If its size > llog chunk_size, then write current chunk to
+                * the update llog. */
+               if (lur->lur_hdr.lrh_len + rec_len + LLOG_MIN_REC_SIZE >
+                   ctxt->loc_chunk_size ||
+                   param_count == record->lur_update_rec.ur_param_count) {
+                       lur->lur_update_rec.ur_update_count =
+                               update_count > last_update_count ?
+                               update_count - last_update_count : 0;
+                       lur->lur_update_rec.ur_param_count = param_count -
+                                                            last_param_count;
+
+                       memcpy(&lur->lur_update_rec.ur_ops, start,
+                              (unsigned long)(src - start));
+                       if (last_update_count != 0)
+                               lur->lur_update_rec.ur_flags |=
+                                               UPDATE_RECORD_CONTINUE;
+
+                       update_records_dump(&lur->lur_update_rec, D_INFO, true);
+                       lur->lur_hdr.lrh_len = llog_update_record_size(lur);
+                       LASSERT(lur->lur_hdr.lrh_len <= ctxt->loc_chunk_size);
+
+                       OBD_ALLOC_PTR(stc);
+                       if (stc == NULL)
+                               GOTO(llog_put, rc = -ENOMEM);
+                       INIT_LIST_HEAD(&stc->stc_list);
+
+                       rc = llog_add(env, ctxt->loc_handle,
+                                     &lur->lur_hdr,
+                                     &stc->stc_cookie, sub_th->st_sub_th);
+
+                       CDEBUG(D_INFO, "%s: Add update log "DOSTID":%u"
+                              " rc = %d\n", dt->dd_lu_dev.ld_obd->obd_name,
+                              POSTID(&stc->stc_cookie.lgc_lgl.lgl_oi),
+                              stc->stc_cookie.lgc_index, rc);
+
+                       if (rc > 0) {
+                               list_add(&stc->stc_list,
+                                        &sub_th->st_cookie_list);
+                               rc = 0;
+                       } else {
+                               OBD_FREE_PTR(stc);
+                               GOTO(llog_put, rc);
+                       }
+
+                       last_update_count = update_count;
+                       last_param_count = param_count;
+                       start = src;
+                       lur->lur_update_rec.ur_update_count = 0;
+                       lur->lur_update_rec.ur_param_count = 0;
+                       lur->lur_hdr.lrh_len = llog_update_record_size(lur);
+               }
+
+               src = next;
+               lur->lur_hdr.lrh_len += cfs_size_round(rec_len);
+               if (update_count < record->lur_update_rec.ur_update_count)
+                       update_count++;
+               else if (param_count < record->lur_update_rec.ur_param_count)
+                       param_count++;
+               else
+                       break;
+       } while (1);
+
+llog_put:
+       if (lur != NULL)
+               OBD_FREE_LARGE(lur, ctxt->loc_chunk_size);
+       llog_ctxt_put(ctxt);
 
        RETURN(rc);
 }
@@ -308,6 +443,7 @@ struct sub_thandle *create_sub_thandle(struct top_multiple_thandle *tmt,
                RETURN(ERR_PTR(-ENOMEM));
 
        INIT_LIST_HEAD(&st->st_sub_list);
+       INIT_LIST_HEAD(&st->st_cookie_list);
        st->st_dt = dt_dev;
 
        list_add(&st->st_sub_list, &tmt->tmt_sub_thandle_list);
@@ -315,41 +451,18 @@ struct sub_thandle *create_sub_thandle(struct top_multiple_thandle *tmt,
 }
 
 /**
- * Create sub thandle
- *
- * Create transaction handle for sub_thandle
- *
- * \param[in] env      execution environment
- * \param[in] th       top thandle
- * \param[in] st       sub_thandle
- *
- * \retval             0 if creation succeeds.
- * \retval             negative errno if creation fails.
- */
-int sub_thandle_trans_create(const struct lu_env *env,
-                            struct top_thandle *top_th,
-                            struct sub_thandle *st)
-{
-       struct thandle *sub_th;
-
-       sub_th = dt_trans_create(env, st->st_dt);
-       if (IS_ERR(sub_th))
-               return PTR_ERR(sub_th);
-
-       sub_th->th_top = &top_th->tt_super;
-       st->st_sub_th = sub_th;
-       return 0;
-}
-
-/**
  * sub thandle commit callback
  *
  * Mark the sub thandle to be committed and if all sub thandle are committed
  * notify the top thandle.
  *
- * \param[in] sub_th   sub thandle being committed.
+ * \param[in] env      execution environment
+ * \param[in] sub_th   sub thandle being committed
+ * \param[in] cb       commit callback
+ * \param[in] err      trans result
  */
-static void sub_trans_commit_cb(struct lu_env *env, struct thandle *sub_th,
+static void sub_trans_commit_cb(struct lu_env *env,
+                               struct thandle *sub_th,
                                struct dt_txn_commit_cb *cb, int err)
 {
        struct sub_thandle      *st;
@@ -358,6 +471,7 @@ static void sub_trans_commit_cb(struct lu_env *env, struct thandle *sub_th,
        ENTRY;
 
        /* Check if all sub thandles are committed */
+       spin_lock(&tmt->tmt_sub_lock);
        list_for_each_entry(st, &tmt->tmt_sub_thandle_list, st_sub_list) {
                if (st->st_sub_th == sub_th) {
                        st->st_committed = 1;
@@ -366,6 +480,7 @@ static void sub_trans_commit_cb(struct lu_env *env, struct thandle *sub_th,
                if (!st->st_committed)
                        all_committed = false;
        }
+       spin_unlock(&tmt->tmt_sub_lock);
 
        if (tmt->tmt_result == 0)
                tmt->tmt_result = err;
@@ -390,6 +505,77 @@ static void sub_thandle_register_commit_cb(struct sub_thandle *st,
 }
 
 /**
+ * Sub thandle stop call back
+ *
+ * After sub thandle is stopped, it will call this callback to notify
+ * the top thandle.
+ *
+ * \param[in] th       sub thandle to be stopped
+ * \param[in] rc       result of sub trans
+ */
+static void sub_trans_stop_cb(struct lu_env *env,
+                             struct thandle *sub_th,
+                             struct dt_txn_commit_cb *cb, int err)
+{
+       struct sub_thandle              *st;
+       struct top_multiple_thandle     *tmt = cb->dcb_data;
+       ENTRY;
+
+       list_for_each_entry(st, &tmt->tmt_sub_thandle_list, st_sub_list) {
+               if (st->st_stopped)
+                       continue;
+
+               if (st->st_dt == sub_th->th_dev) {
+                       st->st_stopped = 1;
+                       st->st_result = err;
+                       break;
+               }
+       }
+
+       wake_up(&tmt->tmt_stop_waitq);
+       RETURN_EXIT;
+}
+
+static void sub_thandle_register_stop_cb(struct sub_thandle *st,
+                                        struct top_multiple_thandle *tmt)
+{
+       st->st_stop_dcb.dcb_func = sub_trans_stop_cb;
+       st->st_stop_dcb.dcb_data = tmt;
+       st->st_stop_dcb.dcb_flags = DCB_TRANS_STOP;
+       INIT_LIST_HEAD(&st->st_stop_dcb.dcb_linkage);
+       dt_trans_cb_add(st->st_sub_th, &st->st_stop_dcb);
+}
+
+/**
+ * Create sub thandle
+ *
+ * Create transaction handle for sub_thandle
+ *
+ * \param[in] env      execution environment
+ * \param[in] th       top thandle
+ * \param[in] st       sub_thandle
+ *
+ * \retval             0 if creation succeeds.
+ * \retval             negative errno if creation fails.
+ */
+int sub_thandle_trans_create(const struct lu_env *env,
+                            struct top_thandle *top_th,
+                            struct sub_thandle *st)
+{
+       struct thandle *sub_th;
+
+       sub_th = dt_trans_create(env, st->st_dt);
+       if (IS_ERR(sub_th))
+               return PTR_ERR(sub_th);
+
+       sub_th->th_top = &top_th->tt_super;
+       st->st_sub_th = sub_th;
+
+       sub_th->th_wait_submit = 1;
+       return 0;
+}
+
+/**
  * Create the top transaction.
  *
  * Create the top transaction on the master device. It will create a top
@@ -455,7 +641,8 @@ static int declare_updates_write(const struct lu_env *env,
                if (st->st_sub_th == NULL)
                        continue;
 
-               rc = sub_declare_updates_write(env, record, st->st_sub_th);
+               rc = sub_declare_updates_write(env, record, st->st_sub_th,
+                                              tmt->tmt_record_size);
                if (rc < 0)
                        break;
        }
@@ -469,7 +656,6 @@ static int declare_updates_write(const struct lu_env *env,
  * Assign batchid to the distribute transaction
  *
  * \param[in] tmt      distribute transaction
- *
  */
 static void distribute_txn_assign_batchid(struct top_multiple_thandle *new)
 {
@@ -549,18 +735,16 @@ static int prepare_multiple_node_trans(const struct lu_env *env,
        int                             rc;
        ENTRY;
 
-       /* Prepare the update buffer for recording updates */
-       if (tmt->tmt_update_records != NULL)
-               RETURN(0);
-
-       tur = &update_env_info(env)->uti_tur;
-       rc = check_and_prepare_update_record(env, tur);
-       if (rc < 0)
-               RETURN(rc);
+       if (tmt->tmt_update_records == NULL) {
+               tur = &update_env_info(env)->uti_tur;
+               rc = check_and_prepare_update_record(env, tur);
+               if (rc < 0)
+                       RETURN(rc);
 
-       tmt->tmt_update_records = tur;
+               tmt->tmt_update_records = tur;
+               distribute_txn_assign_batchid(tmt);
+       }
 
-       distribute_txn_assign_batchid(tmt);
        rc = declare_updates_write(env, tmt);
 
        RETURN(rc);
@@ -588,14 +772,13 @@ int top_trans_start(const struct lu_env *env, struct dt_device *master_dev,
        int                             rc = 0;
        ENTRY;
 
-       /* Walk through all of sub transaction to see if it needs to
-        * record updates for this transaction */
        if (tmt == NULL) {
                rc = dt_trans_start(env, top_th->tt_master_sub_thandle->th_dev,
                                    top_th->tt_master_sub_thandle);
                RETURN(rc);
        }
 
+       tmt = top_th->tt_multiple_thandle;
        rc = prepare_multiple_node_trans(env, tmt);
        if (rc < 0)
                RETURN(rc);
@@ -603,17 +786,20 @@ int top_trans_start(const struct lu_env *env, struct dt_device *master_dev,
        list_for_each_entry(st, &tmt->tmt_sub_thandle_list, st_sub_list) {
                if (st->st_sub_th == NULL)
                        continue;
-               st->st_sub_th->th_sync = th->th_sync;
+               if (th->th_sync)
+                       st->st_sub_th->th_sync = th->th_sync;
                st->st_sub_th->th_local = th->th_local;
                st->st_sub_th->th_tags = th->th_tags;
                rc = dt_trans_start(env, st->st_sub_th->th_dev,
                                    st->st_sub_th);
                if (rc != 0)
-                       RETURN(rc);
+                       GOTO(out, rc);
 
+               sub_thandle_register_stop_cb(st, tmt);
                sub_thandle_register_commit_cb(st, tmt);
        }
-
+out:
+       th->th_result = rc;
        RETURN(rc);
 }
 EXPORT_SYMBOL(top_trans_start);
@@ -658,6 +844,57 @@ static bool top_check_write_updates(struct top_thandle *top_th)
 }
 
 /**
+ * Check if top transaction is stopped
+ *
+ * Check if top transaction is stopped, only if all sub transaction
+ * is stopped, then the top transaction is stopped.
+ *
+ * \param [in] top_th  top thandle
+ *
+ * \retval             true if the top transaction is stopped.
+ * \retval             false if the top transaction is not stopped.
+ */
+static bool top_trans_is_stopped(struct top_thandle *top_th)
+{
+       struct top_multiple_thandle     *tmt;
+       struct sub_thandle              *st;
+       bool                    all_stopped = true;
+
+       tmt = top_th->tt_multiple_thandle;
+       list_for_each_entry(st, &tmt->tmt_sub_thandle_list, st_sub_list) {
+               if (!st->st_stopped && st->st_sub_th != NULL) {
+                       all_stopped = false;
+                       break;
+               }
+
+               if (st->st_result != 0 &&
+                   top_th->tt_super.th_result == 0)
+                       top_th->tt_super.th_result = st->st_result;
+       }
+
+       return all_stopped;
+}
+
+/**
+ * Wait result of top transaction
+ *
+ * Wait until all sub transaction get its result.
+ *
+ * \param [in] top_th  top thandle.
+ *
+ * \retval             the result of top thandle.
+ */
+static int top_trans_wait_result(struct top_thandle *top_th)
+{
+       struct l_wait_info      lwi = {0};
+
+       l_wait_event(top_th->tt_multiple_thandle->tmt_stop_waitq,
+                    top_trans_is_stopped(top_th), &lwi);
+
+       RETURN(top_th->tt_super.th_result);
+}
+
+/**
  * Stop the top transaction.
  *
  * Stop the transaction on the master device first, then stop transactions
@@ -684,6 +921,7 @@ int top_trans_stop(const struct lu_env *env, struct dt_device *master_dev,
        ENTRY;
 
        if (likely(top_th->tt_multiple_thandle == NULL)) {
+               LASSERT(master_dev != NULL);
                rc = dt_trans_stop(env, master_dev,
                                   top_th->tt_master_sub_thandle);
                OBD_FREE_PTR(top_th);
@@ -698,12 +936,6 @@ int top_trans_stop(const struct lu_env *env, struct dt_device *master_dev,
         * then these update logs will be sent to other MDTs */
        /* get the master sub thandle */
        master_st = lookup_sub_thandle(tmt, tmt->tmt_master_sub_dt);
-       if (master_st == NULL) {
-               top_multiple_thandle_dump(tmt, D_ERROR);
-               if (th->th_result == 0)
-                       LBUG();
-       }
-
        write_updates = top_check_write_updates(top_th);
 
        /* Step 1: write the updates log on Master MDT */
@@ -722,8 +954,7 @@ int top_trans_stop(const struct lu_env *env, struct dt_device *master_dev,
 
                lur = tur->tur_update_records;
                /* Write updates to the master MDT */
-               rc = sub_updates_write(env, lur, top_th->tt_master_sub_thandle,
-                                      &master_st->st_cookie);
+               rc = sub_updates_write(env, lur, master_st);
 
                /* Cleanup the common parameters in the update records,
                 * master transno callback might add more parameters.
@@ -745,7 +976,8 @@ stop_master_trans:
         * master transno in the update logs to other MDT. */
        if (master_st != NULL && master_st->st_sub_th != NULL) {
                master_st->st_sub_th->th_local = th->th_local;
-               master_st->st_sub_th->th_sync = th->th_sync;
+               if (th->th_sync)
+                       master_st->st_sub_th->th_sync = th->th_sync;
                master_st->st_sub_th->th_tags = th->th_tags;
                master_st->st_sub_th->th_result = th->th_result;
                rc = dt_trans_stop(env, master_st->st_dt, master_st->st_sub_th);
@@ -785,10 +1017,11 @@ stop_master_trans:
                            st->st_sub_th->th_result < 0)
                                continue;
 
-                       rc = sub_updates_write(env, lur, st->st_sub_th,
-                                              &st->st_cookie);
-                       if (rc < 0)
+                       rc = sub_updates_write(env, lur, st);
+                       if (rc < 0) {
+                               th->th_result = rc;
                                break;
+                       }
                }
        }
 
@@ -798,7 +1031,8 @@ stop_other_trans:
                if (st == master_st || st->st_sub_th == NULL)
                        continue;
 
-               st->st_sub_th->th_sync = th->th_sync;
+               if (th->th_sync)
+                       st->st_sub_th->th_sync = th->th_sync;
                st->st_sub_th->th_local = th->th_local;
                st->st_sub_th->th_tags = th->th_tags;
                st->st_sub_th->th_result = th->th_result;
@@ -808,7 +1042,10 @@ stop_other_trans:
                        th->th_result = rc;
        }
 
+       rc = top_trans_wait_result(top_th);
+
        tmt->tmt_result = rc;
+
        /* Balance for the refcount in top_trans_create, Note: if it is NOT
         * multiple node transaction, the top transaction will be destroyed. */
        top_multiple_thandle_put(tmt);
@@ -830,8 +1067,8 @@ EXPORT_SYMBOL(top_trans_stop);
  * \retval     0 if creation succeeds
  * \retval     negative errno if creation fails
  */
-static int top_trans_create_tmt(const struct lu_env *env,
-                               struct top_thandle *top_th)
+int top_trans_create_tmt(const struct lu_env *env,
+                        struct top_thandle *top_th)
 {
        struct top_multiple_thandle *tmt;
 
@@ -843,6 +1080,8 @@ static int top_trans_create_tmt(const struct lu_env *env,
        INIT_LIST_HEAD(&tmt->tmt_sub_thandle_list);
        INIT_LIST_HEAD(&tmt->tmt_commit_list);
        atomic_set(&tmt->tmt_refcount, 1);
+       spin_lock_init(&tmt->tmt_sub_lock);
+       init_waitqueue_head(&tmt->tmt_stop_waitq);
 
        top_th->tt_multiple_thandle = tmt;
 
@@ -862,6 +1101,7 @@ create_sub_thandle_with_thandle(struct top_thandle *top_th,
                return st;
 
        st->st_sub_th = sub_th;
+
        sub_th->th_top = &top_th->tt_super;
        return st;
 }
@@ -883,6 +1123,7 @@ struct thandle *thandle_get_sub_by_dt(const struct lu_env *env,
                                      struct dt_device *sub_dt)
 {
        struct sub_thandle      *st = NULL;
+       struct sub_thandle      *master_st = NULL;
        struct top_thandle      *top_th;
        struct thandle          *sub_th = NULL;
        int                     rc = 0;
@@ -916,21 +1157,29 @@ struct thandle *thandle_get_sub_by_dt(const struct lu_env *env,
                /* Add master sub th to the top trans list */
                tmt->tmt_master_sub_dt =
                        top_th->tt_master_sub_thandle->th_dev;
-               st = create_sub_thandle_with_thandle(top_th,
-                               top_th->tt_master_sub_thandle);
-               if (IS_ERR(st))
-                       GOTO(stop_trans, rc = PTR_ERR(st));
-               top_th->tt_master_sub_thandle->th_sync = 1;
-               top_th->tt_super.th_sync = 1;
+               master_st = create_sub_thandle_with_thandle(top_th,
+                                       top_th->tt_master_sub_thandle);
+               if (IS_ERR(master_st)) {
+                       rc = PTR_ERR(master_st);
+                       master_st = NULL;
+                       GOTO(stop_trans, rc);
+               }
        }
 
        /* create and init sub th to the top trans list */
        st = create_sub_thandle_with_thandle(top_th, sub_th);
+       if (IS_ERR(st)) {
+               rc = PTR_ERR(st);
+               st = NULL;
+               GOTO(stop_trans, rc);
+       }
        st->st_sub_th->th_wait_submit = 1;
 stop_trans:
        if (rc < 0) {
-               if (st != NULL)
-                       OBD_FREE_PTR(st);
+               if (master_st != NULL) {
+                       list_del(&master_st->st_sub_list);
+                       OBD_FREE_PTR(master_st);
+               }
                sub_th->th_result = rc;
                dt_trans_stop(env, sub_dt, sub_th);
                sub_th = ERR_PTR(rc);
@@ -955,7 +1204,15 @@ void top_multiple_thandle_destroy(struct top_multiple_thandle *tmt)
        LASSERT(tmt->tmt_magic == TOP_THANDLE_MAGIC);
        list_for_each_entry_safe(st, tmp, &tmt->tmt_sub_thandle_list,
                                 st_sub_list) {
+               struct sub_thandle_cookie *stc;
+               struct sub_thandle_cookie *tmp;
+
                list_del(&st->st_sub_list);
+               list_for_each_entry_safe(stc, tmp, &st->st_cookie_list,
+                                        stc_list) {
+                       list_del(&stc->stc_list);
+                       OBD_FREE_PTR(stc);
+               }
                OBD_FREE_PTR(st);
        }
        OBD_FREE_PTR(tmt);
@@ -986,23 +1243,28 @@ static int distribute_txn_cancel_records(const struct lu_env *env,
                struct llog_ctxt        *ctxt;
                struct obd_device       *obd;
                struct llog_cookie      *cookie;
+               struct sub_thandle_cookie *stc;
                int rc;
 
-               cookie = &st->st_cookie;
-               if (fid_is_zero(&cookie->lgc_lgl.lgl_oi.oi_fid))
-                       continue;
-
                obd = st->st_dt->dd_lu_dev.ld_obd;
                ctxt = llog_get_context(obd, LLOG_UPDATELOG_ORIG_CTXT);
-               LASSERT(ctxt);
+               if (ctxt == NULL)
+                       continue;
+               list_for_each_entry(stc, &st->st_cookie_list, stc_list) {
+                       cookie = &stc->stc_cookie;
+                       if (fid_is_zero(&cookie->lgc_lgl.lgl_oi.oi_fid))
+                               continue;
 
-               rc = llog_cat_cancel_records(env, ctxt->loc_handle, 1,
-                                            cookie);
+                       rc = llog_cat_cancel_records(env, ctxt->loc_handle, 1,
+                                                    cookie);
+                       CDEBUG(D_HA, "%s: batchid %llu cancel update log "
+                              DOSTID ".%u : rc = %d\n", obd->obd_name,
+                              tmt->tmt_batchid,
+                              POSTID(&cookie->lgc_lgl.lgl_oi),
+                              cookie->lgc_index, rc);
+               }
 
                llog_ctxt_put(ctxt);
-               CDEBUG(D_HA, "%s: batchid %llu cancel update log "DOSTID
-                      ".%u : rc = %d\n", obd->obd_name, tmt->tmt_batchid,
-                      POSTID(&cookie->lgc_lgl.lgl_oi), cookie->lgc_index, rc);
        }
 
        RETURN(0);
@@ -1137,7 +1399,7 @@ distribute_txn_commit_batchid_update(const struct lu_env *env,
        if (rc < 0)
                GOTO(stop, rc);
 
-       dt_trans_cb_add(th, &dtbd->dtbd_cb);
+       rc = dt_trans_cb_add(th, &dtbd->dtbd_cb);
        if (rc < 0)
                GOTO(stop, rc);
 
@@ -1190,8 +1452,11 @@ distribute_txn_commit_batchid_init(const struct lu_env *env,
 
        dt_obj = dt_find_or_create(env, lut->lut_bottom, fid, dof,
                                   attr);
-       if (IS_ERR(dt_obj))
-               GOTO(out_put, rc = PTR_ERR(dt_obj));
+       if (IS_ERR(dt_obj)) {
+               rc = PTR_ERR(dt_obj);
+               dt_obj = NULL;
+               GOTO(out_put, rc);
+       }
 
        tdtd->tdtd_batchid_obj = dt_obj;
 
@@ -1312,23 +1577,10 @@ static int distribute_txn_commit_thread(void *_arg)
                       tdtd->tdtd_committed_batchid);
                /* update globally committed on a storage */
                if (batchid > tdtd->tdtd_committed_batchid) {
-                       distribute_txn_commit_batchid_update(&env, tdtd,
+                       rc = distribute_txn_commit_batchid_update(&env, tdtd,
                                                             batchid);
-                       spin_lock(&tdtd->tdtd_batchid_lock);
-                       if (batchid > tdtd->tdtd_batchid) {
-                               /* This might happen during recovery,
-                                * batchid is initialized as last transno,
-                                * and the batchid in the update records
-                                * on other MDTs might be bigger than
-                                * the batchid, so we need update it to
-                                * avoid duplicate batchid. */
-                               CDEBUG(D_HA, "%s update batchid from "LPU64
-                                      " to "LPU64"\n",
-                                      tdtd->tdtd_lut->lut_obd->obd_name,
-                                      tdtd->tdtd_batchid, batchid);
-                               tdtd->tdtd_batchid = batchid;
-                       }
-                       spin_unlock(&tdtd->tdtd_batchid_lock);
+                       if (rc == 0)
+                               batchid = 0;
                }
                /* cancel the records for committed batchid's */
                /* XXX: should we postpone cancel's till the end of recovery? */
@@ -1443,7 +1695,10 @@ void distribute_txn_fini(const struct lu_env *env,
        wait_event(lut->lut_tdtd_commit_thread.t_ctl_waitq,
                   lut->lut_tdtd_commit_thread.t_flags & SVC_STOPPED);
 
-       if (tdtd->tdtd_batchid_obj != NULL)
+       dtrq_list_destroy(tdtd);
+       if (tdtd->tdtd_batchid_obj != NULL) {
                lu_object_put(env, &tdtd->tdtd_batchid_obj->do_lu);
+               tdtd->tdtd_batchid_obj = NULL;
+       }
 }
 EXPORT_SYMBOL(distribute_txn_fini);